summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--libglusterfs/src/xlator.c4
-rw-r--r--xlators/protocol/server/src/server-helpers.c112
-rw-r--r--xlators/protocol/server/src/server-helpers.h6
-rw-r--r--xlators/protocol/server/src/server-protocol.c39
-rw-r--r--xlators/protocol/server/src/server-protocol.h7
-rw-r--r--xlators/storage/posix/src/posix.c141
-rw-r--r--xlators/storage/posix/src/posix.h11
7 files changed, 305 insertions, 15 deletions
diff --git a/libglusterfs/src/xlator.c b/libglusterfs/src/xlator.c
index 44494a944a6..650a844c912 100644
--- a/libglusterfs/src/xlator.c
+++ b/libglusterfs/src/xlator.c
@@ -870,10 +870,6 @@ xlator_tree_init (xlator_t *xl)
*/
ret = xlator_init_rec (top);
- if (ret == 0 && top->notify) {
- top->notify (top, GF_EVENT_PARENT_UP, NULL);
- }
-
return ret;
}
diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c
index 88dada1516e..0e22f0ba5fa 100644
--- a/xlators/protocol/server/src/server-helpers.c
+++ b/xlators/protocol/server/src/server-helpers.c
@@ -645,13 +645,14 @@ out:
int
-server_connection_cleanup (xlator_t *this, server_connection_t *conn)
+server_connection_cleanup (xlator_t *this, server_connection_t *conn, transport_t *trans)
{
char do_cleanup = 0;
struct _lock_table *ltable = NULL;
fdentry_t *fdentries = NULL;
uint32_t fd_count = 0;
int ret = 0;
+ int i = 0;
if (conn == NULL) {
goto out;
@@ -659,6 +660,12 @@ server_connection_cleanup (xlator_t *this, server_connection_t *conn)
pthread_mutex_lock (&conn->lock);
{
+ for (i = 0; i < TRANSPORTS_PER_SERVER_CONN; i++) {
+ if (conn->transports[i] == trans) {
+ conn->transports[i] = NULL;
+ transport_unref (trans);
+ }
+ }
conn->active_transports--;
if (conn->active_transports == 0) {
if (conn->ltable) {
@@ -850,11 +857,12 @@ out:
server_connection_t *
-server_connection_get (xlator_t *this, const char *id)
+server_connection_get (xlator_t *this, const char *id, transport_t *trans)
{
server_connection_t *conn = NULL;
server_connection_t *trav = NULL;
server_conf_t *conf = NULL;
+ int i = 0;
conf = this->private;
@@ -878,10 +886,30 @@ server_connection_get (xlator_t *this, const char *id)
list_add (&conn->list, &conf->conns);
}
+ if (conn->active_transports == TRANSPORTS_PER_SERVER_CONN) {
+ gf_log (this->name, GF_LOG_DEBUG,
+ "Maximum number of connections allowed is %d",
+ TRANSPORTS_PER_SERVER_CONN);
+ goto unlock;
+ }
+
+ for (i = 0; i < TRANSPORTS_PER_SERVER_CONN; i++) {
+ if (!conn->transports[i])
+ break;
+ }
+
+ if (i == TRANSPORTS_PER_SERVER_CONN) {
+ gf_log (this->name, GF_LOG_DEBUG,
+ "Could not find a vacant slot");
+ goto unlock;
+ }
+
+ conn->transports[i] = transport_ref (trans);
conn->ref++;
conn->active_transports++;
}
+unlock:
pthread_mutex_unlock (&conf->mutex);
return conn;
@@ -918,3 +946,83 @@ server_connection_put (xlator_t *this, server_connection_t *conn)
out:
return;
}
+
+void
+server_child_down (xlator_t *this, xlator_t *bound_xl)
+{
+ server_conf_t *conf = NULL;
+ server_connection_t *trav = NULL;
+ transport_t *trans = NULL;
+ int subvol_idx = 0;
+ int i = 0;
+ xlator_list_t *xltrav = NULL;
+
+ conf = this->private;
+
+ if (conf == NULL)
+ return;
+
+ xltrav = this->children;
+
+ while (xltrav) {
+ if (xltrav->xlator == bound_xl)
+ break;
+ xltrav = xltrav->next;
+ subvol_idx++;
+ }
+ gf_log (this->name, GF_LOG_DEBUG,
+ "subvolume %s(%d) went down", bound_xl->name, subvol_idx);
+
+ conf->subvol_list[subvol_idx] = 0;
+
+ pthread_mutex_lock (&conf->mutex);
+ {
+ if (!list_empty(&conf->conns)) {
+ list_for_each_entry (trav, &conf->conns, list) {
+ if (bound_xl == trav->bound_xl) {
+ gf_log (this->name, GF_LOG_DEBUG,
+ "disonnecting conn=%p", trav);
+ for (i = 0; i < TRANSPORTS_PER_SERVER_CONN; i++)
+ {
+ trans = trav->transports[i];
+ if (trans == NULL)
+ continue;
+ gf_log (this->name, GF_LOG_DEBUG,
+ "disconnecting %p(%d)",
+ trans, i);
+ transport_disconnect (trans);
+ }
+ }
+ }
+ }
+ }
+ pthread_mutex_unlock (&conf->mutex);
+}
+
+void
+server_child_up (xlator_t *this, xlator_t *bound_xl)
+{
+ server_conf_t *conf = NULL;
+ int subvol_idx = 0;
+ xlator_list_t *xltrav = NULL;
+
+ conf = this->private;
+
+ if (conf == NULL)
+ return;
+
+ xltrav = this->children;
+
+ while (xltrav) {
+ if (bound_xl == xltrav->xlator) {
+ break;
+ }
+ subvol_idx++;
+ xltrav = xltrav->next;
+ }
+
+ gf_log (this->name, GF_LOG_DEBUG,
+ "subvolume %s(%d) came up", bound_xl->name, subvol_idx);
+
+ conf->subvol_list[subvol_idx] = 1;
+}
diff --git a/xlators/protocol/server/src/server-helpers.h b/xlators/protocol/server/src/server-helpers.h
index 867035d3334..5c584b00aa7 100644
--- a/xlators/protocol/server/src/server-helpers.h
+++ b/xlators/protocol/server/src/server-helpers.h
@@ -69,4 +69,10 @@ gf_direntry_to_bin (dir_entry_t *head, char *bufferp);
void
server_print_request (call_frame_t *frame);
+void
+server_child_up (xlator_t *this, xlator_t *bound_xl);
+
+void
+server_child_down (xlator_t *this, xlator_t *bound_xl);
+
#endif /* __SERVER_HELPERS_H__ */
diff --git a/xlators/protocol/server/src/server-protocol.c b/xlators/protocol/server/src/server-protocol.c
index 0528699acbe..a7f6294f387 100644
--- a/xlators/protocol/server/src/server-protocol.c
+++ b/xlators/protocol/server/src/server-protocol.c
@@ -5654,6 +5654,8 @@ mop_setvolume (call_frame_t *frame, xlator_t *bound_xl,
char *volfile_key = NULL;
uint32_t checksum = 0;
int32_t lru_limit = 1024;
+ xlator_list_t *xltrav = NULL;
+ int subvol_idx = 0;
params = dict_new ();
reply = dict_new ();
@@ -5695,7 +5697,7 @@ mop_setvolume (call_frame_t *frame, xlator_t *bound_xl,
}
- conn = server_connection_get (frame->this, process_uuid);
+ conn = server_connection_get (frame->this, process_uuid, trans);
if (trans->xl_private != conn)
trans->xl_private = conn;
@@ -5862,6 +5864,22 @@ mop_setvolume (call_frame_t *frame, xlator_t *bound_xl,
ret = dict_set_uint64 (reply, "transport-ptr",
((uint64_t) (long) trans));
+ xltrav = frame->this->children;
+ while (xltrav) {
+ if (xltrav->xlator == xl)
+ break;
+ xltrav = xltrav->next;
+ subvol_idx++;
+ }
+
+ if (conf->subvol_list[subvol_idx] == 0) {
+ gf_log (xl->name, GF_LOG_DEBUG,
+ "subvolume %d down (filesystem not accesible), failed to setvolume", subvol_idx);
+ op_ret = -1;
+ op_errno = ENOTCONN;
+ goto fail;
+ }
+
fail:
dict_len = dict_serialized_length (reply);
if (dict_len < 0) {
@@ -6550,6 +6568,8 @@ init (xlator_t *this)
server_conf_t *conf = NULL;
data_t *data = NULL;
data_t *trace = NULL;
+ int i = 0;
+ xlator_list_t *xltrav = NULL;
if (this->children == NULL) {
gf_log (this->name, GF_LOG_ERROR,
@@ -6635,6 +6655,15 @@ init (xlator_t *this)
}
}
+ xltrav = this->children;
+
+ while (xltrav) {
+ i++;
+ xltrav = xltrav->next;
+ }
+
+ conf->subvol_list = calloc (i, sizeof (char));
+
#ifndef GF_DARWIN_HOST_OS
{
struct rlimit lim;
@@ -6736,6 +6765,12 @@ notify (xlator_t *this, int32_t event, void *data, ...)
}
switch (event) {
+ case GF_EVENT_CHILD_DOWN:
+ server_child_down (this, data);
+ break;
+ case GF_EVENT_CHILD_UP:
+ server_child_up (this, data);
+ break;
case GF_EVENT_POLLIN:
ret = protocol_server_pollin (this, trans);
break;
@@ -6756,7 +6791,7 @@ notify (xlator_t *this, int32_t event, void *data, ...)
* FIXME: shouldn't we check for return value?
* what should be done if cleanup fails?
*/
- server_connection_cleanup (this, trans->xl_private);
+ server_connection_cleanup (this, trans->xl_private, trans);
}
}
break;
diff --git a/xlators/protocol/server/src/server-protocol.h b/xlators/protocol/server/src/server-protocol.h
index 78bc138279a..73e7f78911b 100644
--- a/xlators/protocol/server/src/server-protocol.h
+++ b/xlators/protocol/server/src/server-protocol.h
@@ -37,6 +37,7 @@
#define DEFAULT_BLOCK_SIZE 4194304 /* 4MB */
#define DEFAULT_VOLUME_FILE_PATH CONFDIR "/glusterfs.vol"
+#define TRANSPORTS_PER_SERVER_CONN 2
typedef struct _server_state server_state_t;
@@ -64,6 +65,7 @@ struct _server_connection {
char *id;
int ref;
int active_transports;
+ transport_t *transports[TRANSPORTS_PER_SERVER_CONN];
pthread_mutex_t lock;
char disconnected;
fdtable_t *fdtable;
@@ -75,7 +77,7 @@ typedef struct _server_connection server_connection_t;
server_connection_t *
-server_connection_get (xlator_t *this, const char *id);
+server_connection_get (xlator_t *this, const char *id, transport_t *trans);
void
server_connection_put (xlator_t *this, server_connection_t *conn);
@@ -84,7 +86,7 @@ int
server_connection_destroy (xlator_t *this, server_connection_t *conn);
int
-server_connection_cleanup (xlator_t *this, server_connection_t *conn);
+server_connection_cleanup (xlator_t *this, server_connection_t *conn, transport_t *trans);
int
server_nop_cbk (call_frame_t *frame, void *cookie,
@@ -107,6 +109,7 @@ typedef struct {
pthread_mutex_t mutex;
struct list_head conns;
gf_boolean_t verify_volfile_checksum;
+ char *subvol_list;
gf_boolean_t trace;
} server_conf_t;
diff --git a/xlators/storage/posix/src/posix.c b/xlators/storage/posix/src/posix.c
index 1ff9a06f9fe..c9342ac4d69 100644
--- a/xlators/storage/posix/src/posix.c
+++ b/xlators/storage/posix/src/posix.c
@@ -4703,6 +4703,119 @@ posix_inode (xlator_t *this)
return 0;
}
+void
+posix_fsping_timer_expired (void *data)
+{
+ xlator_t *this = NULL;
+ struct posix_private *priv = NULL;
+
+ this = data;
+ priv = this->private;
+
+ pthread_mutex_lock (&priv->mutex);
+ {
+ if (priv->fsping_timer) {
+ gf_timer_call_cancel (this->ctx,
+ priv->fsping_timer);
+ priv->fsping_timer = NULL;
+ }
+
+ if (priv->fs_state) {
+ priv->fs_state = 0;
+ default_notify (this, GF_EVENT_CHILD_DOWN, NULL);
+ }
+ }
+ pthread_mutex_unlock (&priv->mutex);
+}
+
+void
+posix_fsping (void *arg);
+
+void *
+posix_fsping_statvfs (void *arg)
+{
+ int ret = -1;
+ xlator_t *this = NULL;
+ char *root_path = NULL;
+ struct statvfs buf = {0, };
+ struct posix_private *priv = NULL;
+ struct timeval delta = {0, };
+
+ this = arg;
+ priv = this->private;
+ root_path = POSIX_BASE_PATH (this);
+
+ ret = statvfs (root_path, &buf);
+
+ pthread_mutex_lock (&priv->mutex);
+ {
+ if (priv->fsping_timer) {
+ gf_timer_call_cancel (this->ctx,
+ priv->fsping_timer);
+ priv->fsping_timer = NULL;
+ }
+ if (ret == 0) {
+ if (priv->fs_state == 0) {
+ priv->fs_state = 1;
+ default_notify (this, GF_EVENT_CHILD_UP,
+ NULL);
+ }
+ } else {
+ if (priv->fs_state) {
+ priv->fs_state = 0;
+ default_notify (this, GF_EVENT_CHILD_DOWN,
+ NULL);
+ }
+ }
+ }
+ pthread_mutex_unlock (&priv->mutex);
+
+ delta.tv_sec = POSIX_FSPING_SLEEP_TIME;
+ priv->fsping_timer =
+ gf_timer_call_after (this->ctx,
+ delta,
+ posix_fsping,
+ (void *) this);
+ if (priv->fsping_timer == NULL) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "unable to register timer");
+ }
+ return NULL;
+}
+
+void
+posix_fsping (void *arg)
+{
+ xlator_t *this = NULL;
+ struct posix_private *priv = NULL;
+ struct timeval delta = {0, };
+
+ this = arg;
+ priv = this->private;
+
+ delta.tv_sec = priv->fsping_timeout;
+ delta.tv_usec = 0;
+
+ if (priv->fsping_timer) {
+ gf_timer_call_cancel (this->ctx,
+ priv->fsping_timer);
+ }
+ priv->fsping_timer =
+ gf_timer_call_after (this->ctx,
+ delta,
+ posix_fsping_timer_expired,
+ (void *) this);
+
+ if (priv->fsping_timer == NULL) {
+ gf_log (this->name, GF_LOG_ERROR,
+ "unable to register timer");
+ /*FIXME: handle error*/
+ }
+ pthread_create (&priv->fsping,
+ NULL,
+ posix_fsping_statvfs,
+ this);
+}
int32_t
posix_rchecksum (call_frame_t *frame, xlator_t *this,
@@ -4780,12 +4893,15 @@ notify (xlator_t *this,
void *data,
...)
{
+ struct posix_private *priv = NULL;
+
+ priv = this->private;
+
switch (event)
{
case GF_EVENT_PARENT_UP:
{
- /* Tell the parent that posix xlator is up */
- default_notify (this, GF_EVENT_CHILD_UP, data);
+ posix_fsping ((void *)this);
}
break;
default:
@@ -4809,9 +4925,9 @@ init (xlator_t *this)
data_t * dir_data = NULL;
data_t * tmp_data = NULL;
uint64_t time64 = 0;
-
- int dict_ret = 0;
- int32_t janitor_sleep;
+ int dict_ret = -1;
+ int fsping_timeout = -1;
+ int32_t janitor_sleep;
dir_data = dict_get (this->options, "directory");
@@ -4908,6 +5024,7 @@ init (xlator_t *this)
strcat (_private->trash_path, "/" GF_REPLICATE_TRASH_DIR);
LOCK_INIT (&_private->lock);
+ pthread_mutex_init (&_private->mutex, NULL);
ret = gethostname (_private->hostname, 256);
if (ret < 0) {
@@ -4923,6 +5040,17 @@ init (xlator_t *this)
_private->max_write = 1;
}
+ _private->fsping_timeout = POSIX_FSPING_TIMEOUT;
+ dict_ret = dict_get_int32 (this->options,
+ "fsping-timeout",
+ &fsping_timeout);
+
+ if (dict_ret == 0) {
+ _private->fsping_timeout = fsping_timeout;
+ }
+ gf_log (this->name, GF_LOG_DEBUG,
+ "fsping-timeout set to %d", _private->fsping_timeout);
+
_private->export_statfs = 1;
tmp_data = dict_get (this->options, "export-statfs-size");
if (tmp_data) {
@@ -5056,6 +5184,7 @@ fini (xlator_t *this)
{
struct posix_private *priv = this->private;
sys_lremovexattr (priv->base_path, "trusted.glusterfs.test");
+ pthread_mutex_destroy (&priv->mutex);
FREE (priv);
return;
}
@@ -5132,6 +5261,8 @@ struct volume_options options[] = {
.type = GF_OPTION_TYPE_BOOL },
{ .key = {"span-devices"},
.type = GF_OPTION_TYPE_INT },
+ { .key = {"fsping-timeout"},
+ .type = GF_OPTION_TYPE_INT },
{ .key = {"background-unlink"},
.type = GF_OPTION_TYPE_BOOL },
{ .key = {"janitor-sleep-duration"},
diff --git a/xlators/storage/posix/src/posix.h b/xlators/storage/posix/src/posix.h
index f92e256fbc0..92fe8e2515c 100644
--- a/xlators/storage/posix/src/posix.h
+++ b/xlators/storage/posix/src/posix.h
@@ -29,6 +29,7 @@
#include <unistd.h>
#include <sys/types.h>
#include <dirent.h>
+#include <pthread.h>
#include <time.h>
#ifdef linux
@@ -50,6 +51,7 @@
#include "xlator.h"
#include "inode.h"
#include "compat.h"
+#include "timer.h"
/**
* posix_fd - internal structure common to file and directory fd's
@@ -70,6 +72,7 @@ struct posix_private {
int32_t base_path_length;
gf_lock_t lock;
+ pthread_mutex_t mutex;
char hostname[256];
/* Statistics, provides activity of the server */
@@ -117,6 +120,11 @@ struct posix_private {
int num_devices_to_span;
dev_t *st_device;
+ pthread_t fsping;
+ gf_timer_t *fsping_timer;
+ int fsping_timeout;
+ int fs_state;
+
/* a global generation number sequence is used to assign generation numbers in
sequence.
*/
@@ -129,6 +137,9 @@ struct posix_private {
char * trash_path;
};
+#define POSIX_FSPING_SLEEP_TIME 10
+#define POSIX_FSPING_TIMEOUT 10
+
#define POSIX_BASE_PATH(this) (((struct posix_private *)this->private)->base_path)
#define POSIX_BASE_PATH_LEN(this) (((struct posix_private *)this->private)->base_path_length)