diff options
| author | Mohammed Junaid Ahmed <junaid@gluster.com> | 2011-02-14 19:27:34 +0000 | 
|---|---|---|
| committer | Anand V. Avati <avati@dev.gluster.com> | 2011-02-15 21:44:27 -0800 | 
| commit | 5a6720f1ee4ad6d96dae23742315c8ef35555a3b (patch) | |
| tree | 1741c6a5fdf6431011223c6d07363da3b5c7c337 | |
| parent | 98cfaa15d8c51728ea4d3555667328b94ef497c1 (diff) | |
syncdaemon: Moved the gsync start code to cli.
Signed-off-by: Junaid <junaid@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>
BUG: 1570 (geosync related changes)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1570
| -rw-r--r-- | cli/src/Makefile.am | 3 | ||||
| -rw-r--r-- | cli/src/cli3_1-cops.c | 176 | ||||
| -rw-r--r-- | rpc/xdr/src/cli1-xdr.c | 2 | ||||
| -rw-r--r-- | rpc/xdr/src/cli1-xdr.h | 1 | ||||
| -rw-r--r-- | rpc/xdr/src/cli1.x | 1 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/Makefile.am | 2 | ||||
| -rw-r--r-- | xlators/mgmt/glusterd/src/glusterd-op-sm.c | 232 | 
7 files changed, 261 insertions, 156 deletions
diff --git a/cli/src/Makefile.am b/cli/src/Makefile.am index 595cfad9d..aa66ed140 100644 --- a/cli/src/Makefile.am +++ b/cli/src/Makefile.am @@ -15,7 +15,8 @@ AM_CFLAGS = -fPIC -Wall -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS)\  	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src\  	-I$(top_srcdir)/rpc/xdr/src\  	-DDATADIR=\"$(localstatedir)\" \ -	-DCONFDIR=\"$(sysconfdir)/glusterfs\" $(GF_GLUSTERFS_CFLAGS) +	-DCONFDIR=\"$(sysconfdir)/glusterfs\" $(GF_GLUSTERFS_CFLAGS)\ +	-DGSYNCD_PREFIX=\"$(libexecdir)\"  CLEANFILES = diff --git a/cli/src/cli3_1-cops.c b/cli/src/cli3_1-cops.c index 9aa4c2fb5..55c69f88f 100644 --- a/cli/src/cli3_1-cops.c +++ b/cli/src/cli3_1-cops.c @@ -2397,7 +2397,7 @@ out:  }  int -gsync_get_command (gf1_cli_gsync_set_rsp rsp) +gf_cli3_1_gsync_get_command (gf1_cli_gsync_set_rsp rsp)  {          char  cmd[1024] = {0,}; @@ -2407,19 +2407,19 @@ gsync_get_command (gf1_cli_gsync_set_rsp rsp)          if (!rsp.gsync_prefix || !rsp.master || !rsp.slave)                  return -1; -        if (rsp.type == GF_GSYNC_OPTION_TYPE_CONFIG_GET) { +        if (rsp.config_type == GF_GSYNC_OPTION_TYPE_CONFIG_GET) {                  if (!rsp.op_name)                          return -1; -                snprintf (cmd, 1024, "%s/gsyncd.py %s %s --config-get %s ", +                snprintf (cmd, 1024, "%s/gsyncd %s %s --config-get %s ",                            rsp.gsync_prefix, rsp.master, rsp.slave,                            rsp.op_name);                  system (cmd);                  goto out;          } -        if (rsp.type == GF_GSYNC_OPTION_TYPE_CONFIG_GET_ALL) { -                snprintf (cmd, 1024, "%s/gsyncd.py %s %s --config-get-all ", -                        rsp.gsync_prefix, rsp.master, rsp.slave); +        if (rsp.config_type == GF_GSYNC_OPTION_TYPE_CONFIG_GET_ALL) { +                snprintf (cmd, 1024, "%s/gsyncd %s %s --config-get-all ", +                          rsp.gsync_prefix, rsp.master, rsp.slave);                  system (cmd); @@ -2430,6 +2430,149 @@ out:  }  int +gf_cli3_1_gsync_get_pid_file (char *pidfile, char *master, char *slave) +{ +        int     ret      = -1; +        int     i        = 0; +        char    str[256] = {0, }; + +        GF_VALIDATE_OR_GOTO ("gsync", pidfile, out); +        GF_VALIDATE_OR_GOTO ("gsync", master, out); +        GF_VALIDATE_OR_GOTO ("gsync", slave, out); + +        i = 0; +        //change '/' to '-' +        while (slave[i]) { +                (slave[i] == '/') ? (str[i] = '-') : (str[i] = slave[i]); +                i++; +        } + +        ret = snprintf (pidfile, 1024, "/etc/glusterd/gsync/%s/%s.pid", +                        master, str); +        if (ret <= 0) { +                ret = -1; +                goto out; +        } + +        ret = 0; +out: +        return ret; +} + +/* status: 0 when gsync is running + * -1 when not running + */ +int +gf_cli3_1_gsync_status (char *master, char *slave, +                        char *pidfile, int *status) +{ +        int     ret             = -1; +        FILE    *file           = NULL; + +        GF_VALIDATE_OR_GOTO ("gsync", master, out); +        GF_VALIDATE_OR_GOTO ("gsync", slave, out); +        GF_VALIDATE_OR_GOTO ("gsync", pidfile, out); +        GF_VALIDATE_OR_GOTO ("gsync", status, out); + +        file = fopen (pidfile, "r+"); +        if (file) { +                //ret = lockf (fileno (file), F_TLOCK, 0); +                //if (ret == 0) { +                //        lockf (fileno (file), F_ULOCK, 0); +                //        *status = -1; +                //} +                //else +                *status = 0; +        } else +                *status = -1; + +out: +        return ret; +} + +int +gf_cli3_1_start_gsync (char *master, char *slave) +{ +        int32_t         ret     = -1; +        int32_t         status  = 0; +        char            cmd[1024] = {0,}; +        char            pidfile[1024] = {0,}; + +        ret = gf_cli3_1_gsync_get_pid_file (pidfile, master, slave); +        if (ret == -1) { +                ret = -1; +                gf_log ("", GF_LOG_WARNING, "failed to construct the " +                        "pidfile string"); +                goto out; +        } + +        ret = gf_cli3_1_gsync_status (master, slave, pidfile, &status); +        if ((ret == 0 && status == 0)) { +                gf_log ("", GF_LOG_WARNING, "gsync %s:%s" +                        "already started", master, slave); + +                cli_out ("gsyncd is already running"); + +                ret = -1; +                goto out; +        } + +        unlink (pidfile); + +        ret = snprintf (cmd, 1024, "mkdir -p /etc/glusterd/gsync/%s", +                        master); +        if (ret <= 0) { +                ret = -1; +                gf_log ("", GF_LOG_WARNING, "failed to construct the " +                        "pid path"); +                goto out; +        } + +        ret = system (cmd); +        if (ret == -1) { +                gf_log ("", GF_LOG_WARNING, "failed to create the " +                        "pid path for %s %s", master, slave); +                goto out; +        } + +        memset (cmd, 0, sizeof (cmd)); +        ret = snprintf (cmd, 1024, GSYNCD_PREFIX "/gsyncd %s %s " +                        "--config-set pid-file %s", master, slave, pidfile); +        if (ret <= 0) { +                ret = -1; +                gf_log ("", GF_LOG_WARNING, "failed to construct the  " +                        "config set command for %s %s", master, slave); +                goto out; +        } + +        ret = system (cmd); +        if (ret == -1) { +                gf_log ("", GF_LOG_WARNING, "failed to set the pid " +                        "option for %s %s", master, slave); +                goto out; +        } + +        memset (cmd, 0, sizeof (cmd)); +        ret = snprintf (cmd, 1024, GSYNCD_PREFIX "/gsyncd " +                        "%s %s", master, slave); +        if (ret <= 0) { +                ret = -1; +                goto out; +        } + +        ret = system (cmd); +        if (ret == -1) +                goto out; + +        cli_out ("gsync started"); +        ret = 0; + +out: + +        return ret; +} + +int  gf_cli3_1_gsync_set_cbk (struct rpc_req *req, struct iovec *iov,                           int count, void *myframe)  { @@ -2448,14 +2591,19 @@ gf_cli3_1_gsync_set_cbk (struct rpc_req *req, struct iovec *iov,                  goto out;          } -        if (rsp.op_errstr) -                cli_out ("%s", rsp.op_errstr); -        else if (rsp.op_ret) -                cli_out ("command unsuccessful"); -        else -                cli_out ("command executed successfully"); - -        gsync_get_command (rsp); +        if (rsp.op_ret) { +                cli_out ("%s", rsp.op_errstr ? rsp.op_errstr : +                         "command unsuccessful"); +                goto out; +        } +        else { +                if (rsp.type == GF_GSYNC_OPTION_TYPE_START) +                        ret = gf_cli3_1_start_gsync (rsp.master, rsp.slave); +                else if (rsp.config_type == GF_GSYNC_OPTION_TYPE_CONFIG_GET_ALL) +                        ret = gf_cli3_1_gsync_get_command (rsp); +                else +                        cli_out ("command executed successfully"); +        }  out:          ret = rsp.op_ret; diff --git a/rpc/xdr/src/cli1-xdr.c b/rpc/xdr/src/cli1-xdr.c index 3a2f6a421..453513aa3 100644 --- a/rpc/xdr/src/cli1-xdr.c +++ b/rpc/xdr/src/cli1-xdr.c @@ -669,6 +669,8 @@ xdr_gf1_cli_gsync_set_rsp (XDR *xdrs, gf1_cli_gsync_set_rsp *objp)                  return FALSE;          if (!xdr_int (xdrs, &objp->type))                  return FALSE; +        if (!xdr_int (xdrs, &objp->config_type)) +                return FALSE;          if (!xdr_string (xdrs, &objp->op_name, ~0))                  return FALSE;          if (!xdr_string (xdrs, &objp->master, ~0)) diff --git a/rpc/xdr/src/cli1-xdr.h b/rpc/xdr/src/cli1-xdr.h index c47494312..302b1f8ab 100644 --- a/rpc/xdr/src/cli1-xdr.h +++ b/rpc/xdr/src/cli1-xdr.h @@ -416,6 +416,7 @@ struct gf1_cli_gsync_set_rsp {          int op_errno;          char *op_errstr;          int   type; +        int   config_type;          char *op_name;          char *master;          char *slave; diff --git a/rpc/xdr/src/cli1.x b/rpc/xdr/src/cli1.x index 97c50f6fd..0e18c6ab9 100644 --- a/rpc/xdr/src/cli1.x +++ b/rpc/xdr/src/cli1.x @@ -301,6 +301,7 @@ struct gf1_cli_gsync_set_rsp {          int     op_errno;          string  op_errstr<>;          int     type; +        int     config_type;          string  op_name<>;          string  master<>;          string  slave<>; diff --git a/xlators/mgmt/glusterd/src/Makefile.am b/xlators/mgmt/glusterd/src/Makefile.am index 2336000b5..b1859de5d 100644 --- a/xlators/mgmt/glusterd/src/Makefile.am +++ b/xlators/mgmt/glusterd/src/Makefile.am @@ -16,7 +16,7 @@ AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\  	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)\  	-I$(rpclibdir) -L$(xlatordir)/ -I$(CONTRIBDIR)/rbtree -I$(top_srcdir)/rpc/xdr/src\  	-I$(top_srcdir)/rpc/rpc-lib/src -I$(CONTRIBDIR)/uuid -DGFS_PREFIX=\"$(prefix)\" \ -	-DDATADIR=\"$(localstatedir)\" -DGSYNC_PREFIX=\"$(libexecdir)\" +	-DDATADIR=\"$(localstatedir)\" -DGSYNCD_PREFIX=\"$(libexecdir)\"  CLEANFILES = diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c index 5adecef83..770443b9f 100644 --- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c +++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c @@ -54,8 +54,6 @@  #define glusterd_op_start_volume_args_get(req, dict, volname, flags) \          glusterd_op_stop_volume_args_get (req, dict, volname, flags) -#define GSYNCD_PREFIX GSYNC_PREFIX"/python/syncdaemon/" -  static struct list_head gd_op_sm_queue;  pthread_mutex_t       gd_op_sm_lock;  glusterd_op_info_t    opinfo = {{0},}; @@ -1904,29 +1902,63 @@ volname_from_master (char *master)          return gf_strdup (master+1);  } -/* status: return 0 when gsync is running - * return -1 when not running - */  int -gsync_status (char *master, char *slave) +gsync_get_pid_file (char *pidfile, char *master, char *slave)  { -        int     ret             = 0; -        char    pidfile[1024]   = {0,}; -        FILE    *file            = NULL; +        int     ret      = -1; +        int     i        = 0; +        char    str[256] = {0, }; + +        GF_VALIDATE_OR_GOTO ("gsync", pidfile, out); +        GF_VALIDATE_OR_GOTO ("gsync", master, out); +        GF_VALIDATE_OR_GOTO ("gsync", slave, out); + +        i = 0; +        //change '/' to '-' +        while (slave[i]) { +                (slave[i] == '/') ? (str[i] = '-') : (str[i] = slave[i]); +                i++; +        }          ret = snprintf (pidfile, 1024, "/etc/glusterd/gsync/%s/%s.pid", -                        master, slave); +                        master, str);          if (ret <= 0) {                  ret = -1;                  goto out;          } +        ret = 0; +out: +        return ret; +} + +/* status: return 0 when gsync is running + * return -1 when not running + */ +int +gsync_status (char *master, char *slave, int *status) +{ +        int     ret             = -1; +        char    pidfile[1024]   = {0,}; +        FILE    *file           = NULL; + +        GF_VALIDATE_OR_GOTO ("gsync", master, out); +        GF_VALIDATE_OR_GOTO ("gsync", slave, out); +        GF_VALIDATE_OR_GOTO ("gsync", status, out); + +        ret = gsync_get_pid_file (pidfile, master, slave); +        if (ret == -1) +                goto out; +          file = fopen (pidfile, "r+");          if (file) { -                ret = lockf (fileno (file), F_TLOCK, 0); -                if (ret && ((EAGAIN == errno) || (EACCES == errno))) -                        ret = 0; -        } +               // ret = lockf (fileno (file), F_TEST, 0); +                //if (ret == 0) +                //*status = -1; +                //else +                *status = 0; +        } else +                *status = -1;  out:          return ret;  } @@ -2017,6 +2049,7 @@ glusterd_op_stage_gsync_set (gd1_mgmt_stage_op_req *req, char **op_errstr)  {          int             ret     = 0;          int             type    = 0; +        int             status  = 0;          dict_t          *dict   = NULL;          char            *volname = NULL;          char            *master  = NULL; @@ -2095,24 +2128,36 @@ glusterd_op_stage_gsync_set (gd1_mgmt_stage_op_req *req, char **op_errstr)                          goto out;                  }                  //check if the gsync is already started -                ret = gsync_status (master, slave); -                if (ret == 0) { +                ret = gsync_status (master, slave, &status); +                if (ret == 0 && status == 0) {                          gf_log ("", GF_LOG_WARNING, "gsync already started");                          *op_errstr = gf_strdup ("gsync already started");                          ret = -1;                          goto out; +                } else if (ret == -1) { +                        gf_log ("", GF_LOG_WARNING, "gsync start validation " +                                " failed"); +                        *op_errstr = gf_strdup ("command to failed, please " +                                                "check the log file"); +                        goto out;                  }                  ret = 0;                  goto out;          }          if (type == GF_GSYNC_OPTION_TYPE_STOP) { -                ret = gsync_status (master, slave); -                if (ret == -1) { +                ret = gsync_status (master, slave, &status); +                if (ret == 0 && status == -1) {                          gf_log ("", GF_LOG_WARNING, "gsync not running");                          *op_errstr = gf_strdup ("gsync not running");                          ret = -1;                          goto out; +                }  else if (ret == -1) { +                        gf_log ("", GF_LOG_WARNING, "gsync stop validation " +                                " failed"); +                        *op_errstr = gf_strdup ("command to failed, please " +                                                "check the log file"); +                        goto out;                  }                  ret = 0;                  goto out; @@ -3680,127 +3725,26 @@ out:  }  int -gsync_get_pid_file (char *pidfile, char *master, char *slave) -{ -        int     ret      = -1; -        int     i        = 0; -        char    str[256] = {0, }; - -        GF_VALIDATE_OR_GOTO ("gsync", pidfile, out); -        GF_VALIDATE_OR_GOTO ("gsync", master, out); -        GF_VALIDATE_OR_GOTO ("gsync", slave, out); - -        i = 0; -        //change '/' to '-' -        while (slave[i]) { -                (slave[i] == '/') ? (str[i] = '-') : (str[i] = slave[i]); -                i++; -        } - -        ret = snprintf (pidfile, 1024, "/etc/glusterd/gsync/%s/%s.pid", -                        master, str); -        if (ret <= 0) -                ret = -1; - -        ret = 0; -out: -        return 0; -} - -int -start_gsync (char *master, char *slave, char **op_errstr) -{ -        int32_t         ret     = -1; -        char            cmd[1024] = {0,}; -        char            pidfile[1024] = {0,}; - -        ret = gsync_get_pid_file (pidfile, master, slave); -        if (ret == -1) { -                ret = -1; -                gf_log ("", GF_LOG_WARNING, "failed to construct the " -                        "pidfile string"); -                goto out; -        } - -        ret = gsync_status (master, slave); -        if (ret == 0) { -                gf_log ("", GF_LOG_WARNING, "gsync %s:%s" -                        "already started", master, slave); - -                *op_errstr = gf_strdup ("gsync is running"); -                ret = -1; -                goto out; -        } - -        ret = snprintf (cmd, 1024, "mkdir -p /etc/glusterd/gsync/%s", -                        master); -        if (ret <= 0) { -                ret = -1; -                gf_log ("", GF_LOG_WARNING, "failed to construct the " -                        "pid path"); -                goto out; -        } - -        ret = system (cmd); -        if (ret == -1) { -                gf_log ("", GF_LOG_WARNING, "failed to create the " -                        "pid path for %s %s", master, slave); -                goto out; -        } - -        memset (cmd, 0, sizeof (cmd)); -        ret = snprintf (cmd, 1024, GSYNCD_PREFIX "gsyncd.py %s %s " -                        "--config-set pid-file %s", master, slave, pidfile); -        if (ret <= 0) { -                ret = -1; -                gf_log ("", GF_LOG_WARNING, "failed to construct the  " -                        "config set command for %s %s", master, slave); -                goto out; -        } - -        ret = system (cmd); -        if (ret == -1) { -                gf_log ("", GF_LOG_WARNING, "failed to set the pid " -                        "option for %s %s", master, slave); -                goto out; -        } - -        memset (cmd, 0, sizeof (cmd)); -        ret = snprintf (cmd, 1024, GSYNCD_PREFIX "/gsyncd.py " -                        "%s %s", master, slave); -        if (ret <= 0) { -                ret = -1; -                goto out; -        } - -        ret = system (cmd); -        if (ret == -1) -                goto out; - -        ret = 0; - -        *op_errstr = gf_strdup ("gsync started successfully"); - -out: -        if (ret < 0 && *op_errstr == NULL) -                *op_errstr = gf_strdup ("gsync start unsuccessful"); - -        return ret; -} - -int  stop_gsync (char *master, char *slave, char **op_errstr)  {          int32_t         ret     = -1; +        int32_t         status  = 0;          pid_t           pid     = 0;          FILE            *file   = NULL;          char            pidfile[1024] = {0,};          char            buf [256] = {0,}; -        ret = gsync_status (master, slave); -        if (ret == -1) { +        ret = gsync_status (master, slave, &status); +        if (ret == 0 && status == -1) {                  gf_log ("", GF_LOG_WARNING, "gsync is not running");                  *op_errstr = gf_strdup ("gsync is not running"); +                ret = -1; +                goto out; +        } else if (ret == -1) { +                gf_log ("", GF_LOG_WARNING, "gsync stop validation " +                        " failed"); +                *op_errstr = gf_strdup ("command to failed, please " +                                        "check the log file");                  goto out;          } @@ -3823,12 +3767,15 @@ stop_gsync (char *master, char *slave, char **op_errstr)          ret = read (fileno(file), buf, 1024);          if (ret > 0) {                  pid = strtol (buf, NULL, 10); -                ret = kill (pid, SIGKILL); +                ret = kill (pid, SIGTERM);                  if (ret) {                          gf_log ("", GF_LOG_WARNING,                                  "failed to stop gsyncd");                          goto out;                  } +                sleep (0.1); +                kill (pid, SIGTERM); +                unlink (pidfile);          }          ret = 0; @@ -3869,8 +3816,8 @@ gsync_config_set (char *master, char *slave,                  goto out;          } -        ret = snprintf (cmd, 1024, GSYNCD_PREFIX "/gsyncd.py %s %s " -                        "--config-set -%s %s", master, slave, op_name, op_value); +        ret = snprintf (cmd, 1024, GSYNCD_PREFIX "/gsyncd %s %s " +                        "--config-set %s %s", master, slave, op_name, op_value);          if (ret <= 0) {                  gf_log ("", GF_LOG_WARNING, "failed to "                          "construct the gsyncd command"); @@ -3917,8 +3864,8 @@ gsync_config_del (char *master, char *slave,                  goto out;          } -        ret = snprintf (cmd, 4096, GSYNCD_PREFIX "/gsyncd.py %s %s " -                        "config-del %s", master, slave, op_name); +        ret = snprintf (cmd, 4096, GSYNCD_PREFIX "/gsyncd %s %s " +                        "--config-del %s", master, slave, op_name);          if (ret <= 0) {                  gf_log ("", GF_LOG_WARNING, "failed to "                          "construct the gsyncd command"); @@ -3963,8 +3910,8 @@ gsync_config_get (char *master, char *slave,                  goto out;          } -        ret = snprintf (cmd, 4096, GSYNCD_PREFIX "/gsyncd.py %s %s " -                        "config-get %s", master, slave, op_name); +        ret = snprintf (cmd, 4096, GSYNCD_PREFIX "/gsyncd %s %s " +                        "--config-get %s", master, slave, op_name);          if (ret <= 0) {                  gf_log ("", GF_LOG_WARNING, "failed to "                          "construct the gsyncd command"); @@ -3996,7 +3943,7 @@ gsync_config_get_all (char *master, char *slave, char **op_errstr)          int32_t         ret     = -1;          char            cmd[1024] = {0,}; -        ret = snprintf (cmd, 4096, GSYNCD_PREFIX "/gsyncd.py %s %s " +        ret = snprintf (cmd, 4096, GSYNCD_PREFIX "/gsyncd %s %s "                          "config-get-all", master, slave);          if (ret <= 0) {                  gf_log ("", GF_LOG_WARNING, "failed to " @@ -4092,7 +4039,7 @@ gsync_command_exec (dict_t *dict, char **op_errstr)                  goto out;          if (type == GF_GSYNC_OPTION_TYPE_START) { -                ret = start_gsync (master, slave, op_errstr); +                ret = 0;                  goto out;          } @@ -5684,7 +5631,8 @@ glusterd_op_send_cli_response (int32_t op, int32_t op_ret,                          }                  case GD_MGMT_CLI_GSYNC_SET:                          { -                                int     config_type; +                                int     type = 0; +                                int     config_type = 0;                                  char    *str = NULL;                                  char    *master = NULL;                                  char    *slave  = NULL; @@ -5703,10 +5651,14 @@ glusterd_op_send_cli_response (int32_t op, int32_t op_ret,                                                              &str);                                          if (ret == 0)                                                  rsp.op_errstr = gf_strdup (str); +                                        ret = dict_get_int32 (ctx, "type", +                                                              &type); +                                        if (ret == 0) +                                                rsp.type = type;                                          ret = dict_get_int32 (ctx, "config_type",                                                                &config_type);                                          if (ret == 0) -                                                rsp.type = config_type; +                                                rsp.config_type = config_type;                                          ret = dict_get_str (ctx, "master",                                                              &master);                                          if (ret == 0)  | 
