summaryrefslogtreecommitdiffstats
path: root/xlators/cluster/dht
diff options
context:
space:
mode:
authorDan Lambright <dlambrig@redhat.com>2015-03-24 13:59:00 -0400
committerNiels de Vos <ndevos@redhat.com>2015-04-06 02:35:37 -0700
commit37bbcf6861590aa3d980846042855ee4212c89df (patch)
tree3f6ac3e4a627b25e424ab12b789d827858e2e334 /xlators/cluster/dht
parent90f72efed8313efe0410634677f36d0fbdc5eab4 (diff)
cluster/dht: fix tier.c problems found prior to feature freeze
This patch resolves tiering translator issues taken from the list in bug 1203776. These issues have been selected to be fixed first. The rest will be fixed in a subsequent patch (or are not a problem). 3. Replace hardcoded #defines of promote/demote file names 6. Use loc_wipe() in migrate_using_query_file() 9. Only promote/demote files on the same node on which they reside. 14. Replace calloc with GF_CALLOC in tier.c and ensure freeing done properly. 15. Handle if parse_query_str fails 22. Only load gfdb library on server side, remove SQL references from client. Change-Id: I6563b11e58ab2e4c6b1ce44db755781ad6d930fb BUG: 1203776 Signed-off-by: Dan Lambright <dlambrig@redhat.com> Reviewed-on: http://review.gluster.org/9987 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: N Balachandran <nbalacha@redhat.com> Reviewed-by: Niels de Vos <ndevos@redhat.com>
Diffstat (limited to 'xlators/cluster/dht')
-rw-r--r--xlators/cluster/dht/src/Makefile.am10
-rw-r--r--xlators/cluster/dht/src/tier.c243
-rw-r--r--xlators/cluster/dht/src/tier.h10
3 files changed, 205 insertions, 58 deletions
diff --git a/xlators/cluster/dht/src/Makefile.am b/xlators/cluster/dht/src/Makefile.am
index bbe5cdd8d83..a8e1ec0d286 100644
--- a/xlators/cluster/dht/src/Makefile.am
+++ b/xlators/cluster/dht/src/Makefile.am
@@ -27,17 +27,19 @@ nufa_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
switch_la_LDFLAGS = -module -avoid-version
switch_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-tier_la_CFLAGS = $(AM_CFLAGS) $(SQLITE_CFLAGS)
+
tier_la_LDFLAGS = -module -avoid-version
-tier_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la\
- $(top_builddir)/libglusterfs/src/gfdb/libgfdb.la
+tier_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
noinst_HEADERS = dht-common.h dht-mem-types.h dht-messages.h dht-helper.h tier.h\
$(top_builddir)/xlators/lib/src/libxlator.h
AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-I$(top_srcdir)/libglusterfs/src/gfdb \
- -I$(top_srcdir)/xlators/lib/src
+ -I$(top_srcdir)/xlators/lib/src \
+ -DDATADIR=\"$(localstatedir)\" \
+ -DLIBDIR=\"$(libdir)\" \
+ -DLIBGFDB_VERSION=\"$(LIBGFDB_VERSION)\"
CLEANFILES =
diff --git a/xlators/cluster/dht/src/tier.c b/xlators/cluster/dht/src/tier.c
index 51a6a8340e9..bc969422bda 100644
--- a/xlators/cluster/dht/src/tier.c
+++ b/xlators/cluster/dht/src/tier.c
@@ -12,6 +12,7 @@
#define _CONFIG_H
#include "config.h"
#endif
+#include <dlfcn.h>
#include "dht-common.h"
#include "tier.h"
@@ -23,8 +24,16 @@ static gfdb_db_type_t dht_tier_db_type = GFDB_SQLITE3;
/*Mutex for updating the data movement stats*/
static pthread_mutex_t dm_stat_mutex = PTHREAD_MUTEX_INITIALIZER;
+static char *promotion_qfile;
+static char *demotion_qfile;
+
+static void *libhandle;
+static gfdb_methods_t gfdb_methods;
+
#define DB_QUERY_RECORD_SIZE 4096
+
+
static int
tier_parse_query_str (char *query_record_str,
char *gfid, char *link_buffer, ssize_t *link_size)
@@ -64,6 +73,51 @@ out:
}
static int
+tier_check_same_node (xlator_t *this, loc_t *loc, gf_defrag_info_t *defrag)
+{
+ int ret = -1;
+ dict_t *dict = NULL;
+ char *uuid_str = NULL;
+ uuid_t node_uuid = {0,};
+
+ GF_VALIDATE_OR_GOTO ("tier", this, out);
+ GF_VALIDATE_OR_GOTO (this->name, loc, out);
+ GF_VALIDATE_OR_GOTO (this->name, defrag, out);
+
+ if (syncop_getxattr (this, loc, &dict, GF_XATTR_NODE_UUID_KEY, NULL)) {
+ gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
+ "Unable to get NODE_UUID_KEY %s %s\n",
+ loc->name, loc->path);
+ goto out;
+ }
+
+ if (dict_get_str (dict, GF_XATTR_NODE_UUID_KEY, &uuid_str) < 0) {
+ gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
+ "Failed to get node-uuid for %s", loc->path);
+ goto out;
+ }
+
+ if (uuid_parse (uuid_str, node_uuid)) {
+ gf_msg (this->name, GF_LOG_INFO, 0, DHT_MSG_LOG_TIER_ERROR,
+ "uuid_parse failed for %s", loc->path);
+ goto out;
+ }
+
+ if (uuid_compare (node_uuid, defrag->node_uuid)) {
+ gf_msg (this->name, GF_LOG_INFO, 0, DHT_MSG_LOG_TIER_STATUS,
+ "%s does not belong to this node", loc->path);
+ goto out;
+ }
+
+ ret = 0;
+out:
+ if (dict)
+ dict_unref(dict);
+
+ return ret;
+}
+
+static int
tier_migrate_using_query_file (void *_args)
{
int ret = -1;
@@ -77,7 +131,7 @@ tier_migrate_using_query_file (void *_args)
char *link_buffer = NULL;
gfdb_query_record_t *query_record = NULL;
gfdb_link_info_t *link_info = NULL;
- struct iatt par_stbuf = {0,};
+ struct iatt par_stbuf = {0,};
struct iatt current = {0,};
loc_t p_loc = {0,};
loc_t loc = {0,};
@@ -104,7 +158,8 @@ tier_migrate_using_query_file (void *_args)
goto out;
}
- query_record->_link_info_str = calloc (DB_QUERY_RECORD_SIZE, 1);
+ query_record->_link_info_str = GF_CALLOC (1, DB_QUERY_RECORD_SIZE,
+ gf_common_mt_char);
if (!query_record->_link_info_str) {
goto out;
}
@@ -249,6 +304,11 @@ tier_migrate_using_query_file (void *_args)
DHT_MSG_LOG_TIER_STATUS, "Tier migrate file %s",
loc.name);
+ if (tier_check_same_node (this, &loc, defrag)) {
+ per_link_status = -1;
+ goto error;
+ }
+
ret = syncop_setxattr (this, &loc, migrate_data, 0);
if (ret) {
gf_msg (this->name, GF_LOG_ERROR, 0,
@@ -259,16 +319,11 @@ tier_migrate_using_query_file (void *_args)
per_link_status = -1;
goto error;
}
- inode_unref (loc.inode);
- inode_unref (loc.parent);
- inode_unref (p_loc.inode);
error:
- if (loc.name) {
- GF_FREE ((char *) loc.name);
- }
- if (loc.path) {
- GF_FREE ((char *) loc.path);
- }
+
+ loc_wipe(&loc);
+ loc_wipe(&p_loc);
+
token_str = NULL;
token_str = strtok (NULL, delimiter);
GF_FREE (link_str);
@@ -292,7 +347,7 @@ per_file_out:
out:
if (link_buffer)
- free (link_buffer);
+ GF_FREE (link_buffer);
gfdb_link_info_fini (&link_info);
if (migrate_data)
dict_unref (migrate_data);
@@ -362,11 +417,11 @@ tier_process_brick_cbk (dict_t *brick_dict, char *key, data_t *value,
"DB Params cannot initialized!");
goto out;
}
- SET_DB_PARAM_TO_DICT(this->name, params_dict, GFDB_SQL_PARAM_DBPATH,
+ SET_DB_PARAM_TO_DICT(this->name, params_dict, gfdb_methods.dbpath,
db_path, ret, out);
/*Get the db connection*/
- conn_node = init_db((void *)params_dict, dht_tier_db_type);
+ conn_node = gfdb_methods.init_db((void *)params_dict, dht_tier_db_type);
if (!conn_node) {
gf_msg (this->name, GF_LOG_ERROR, 0,
DHT_MSG_LOG_TIER_ERROR,
@@ -388,12 +443,14 @@ tier_process_brick_cbk (dict_t *brick_dict, char *key, data_t *value,
if (!gfdb_brick_dict_info->_gfdb_promote) {
if (query_cbk_args->defrag->write_freq_threshold == 0 &&
query_cbk_args->defrag->read_freq_threshold == 0) {
- ret = find_unchanged_for_time(conn_node,
+ ret = gfdb_methods.find_unchanged_for_time (
+ conn_node,
tier_gf_query_callback,
(void *)query_cbk_args,
gfdb_brick_dict_info->time_stamp);
} else {
- ret = find_unchanged_for_time_freq(conn_node,
+ ret = gfdb_methods.find_unchanged_for_time_freq (
+ conn_node,
tier_gf_query_callback,
(void *)query_cbk_args,
gfdb_brick_dict_info->time_stamp,
@@ -406,20 +463,21 @@ tier_process_brick_cbk (dict_t *brick_dict, char *key, data_t *value,
} else {
if (query_cbk_args->defrag->write_freq_threshold == 0 &&
query_cbk_args->defrag->read_freq_threshold == 0) {
- ret = find_recently_changed_files(conn_node,
- tier_gf_query_callback,
- (void *)query_cbk_args,
- gfdb_brick_dict_info->time_stamp);
+ ret = gfdb_methods.find_recently_changed_files (
+ conn_node,
+ tier_gf_query_callback,
+ (void *)query_cbk_args,
+ gfdb_brick_dict_info->time_stamp);
} else {
- ret = find_recently_changed_files_freq(conn_node,
- tier_gf_query_callback,
- (void *)query_cbk_args,
- gfdb_brick_dict_info->time_stamp,
- query_cbk_args->defrag->
- write_freq_threshold,
- query_cbk_args->defrag->
- read_freq_threshold,
- _gf_false);
+ ret = gfdb_methods.find_recently_changed_files_freq (
+ conn_node,
+ tier_gf_query_callback,
+ (void *)query_cbk_args,
+ gfdb_brick_dict_info->time_stamp,
+ query_cbk_args->defrag->
+ write_freq_threshold,
+ query_cbk_args->defrag->read_freq_threshold,
+ _gf_false);
}
}
if (ret) {
@@ -434,7 +492,7 @@ out:
fclose (query_cbk_args->queryFILE);
query_cbk_args->queryFILE = NULL;
}
- fini_db (conn_node);
+ gfdb_methods.fini_db (conn_node);
return ret;
}
@@ -538,7 +596,7 @@ tier_demote (void *args)
/* Migrate files using the query file */
ret = tier_migrate_files_using_qfile (args,
- &query_cbk_args, DEMOTION_QFILE);
+ &query_cbk_args, demotion_qfile);
if (ret)
goto out;
@@ -574,7 +632,7 @@ static void
/* Migrate files using the query file */
ret = tier_migrate_files_using_qfile (args,
&query_cbk_args,
- PROMOTION_QFILE);
+ promotion_qfile);
if (ret)
goto out;
@@ -626,7 +684,9 @@ tier_get_bricklist (xlator_t *xl, dict_t *bricklist)
goto out;
}
- sprintf(db_path, "%s/.glusterfs/%s", rv, db_name);
+ sprintf(db_path, "%s/%s/%s", rv,
+ GF_HIDDEN_PATH,
+ db_name);
if (dict_add_dynstr_with_alloc(bricklist, "brick",
db_path))
goto out;
@@ -855,6 +915,43 @@ dht_methods_t tier_methods = {
.layout_search = tier_search,
};
+static int
+tier_load_externals (xlator_t *this)
+{
+ int ret = -1;
+ char *libpathfull = (LIBDIR "/libgfdb.so." LIBGFDB_VERSION);
+ get_gfdb_methods_t get_gfdb_methods;
+
+ GF_VALIDATE_OR_GOTO("this", this, out);
+
+ libhandle = dlopen (libpathfull, RTLD_NOW);
+ if (!libhandle) {
+ gf_msg(this->name, GF_LOG_ERROR, 0,
+ DHT_MSG_LOG_TIER_ERROR,
+ "Error loading libgfdb.so %s\n", dlerror());
+ ret = -1;
+ goto out;
+ }
+
+ get_gfdb_methods = dlsym (libhandle, "get_gfdb_methods");
+ if (!get_gfdb_methods) {
+ gf_msg(this->name, GF_LOG_ERROR, 0,
+ DHT_MSG_LOG_TIER_ERROR,
+ "Error loading get_gfdb_methods()");
+ ret = -1;
+ goto out;
+ }
+
+ get_gfdb_methods (&gfdb_methods);
+
+ ret = 0;
+
+out:
+ if (ret && libhandle)
+ dlclose (libhandle);
+
+ return ret;
+}
int
tier_init (xlator_t *this)
@@ -889,11 +986,16 @@ tier_init (xlator_t *this)
goto out;
}
- defrag = conf->defrag;
+ /* if instatiated from server side, load db libraries */
+ ret = tier_load_externals(this);
+ if (ret) {
+ gf_msg(this->name, GF_LOG_ERROR, 0,
+ DHT_MSG_LOG_TIER_ERROR,
+ "Could not load externals. Aborting");
+ goto out;
+ }
- GF_OPTION_INIT ("tier-promote-frequency",
- defrag->tier_promote_frequency,
- int32, out);
+ defrag = conf->defrag;
ret = dict_get_int32 (this->options,
"tier-promote-frequency", &freq);
@@ -903,10 +1005,6 @@ tier_init (xlator_t *this)
defrag->tier_promote_frequency = freq;
- GF_OPTION_INIT ("tier-demote-frequency",
- defrag->tier_demote_frequency,
- int32, out);
-
ret = dict_get_int32 (this->options,
"tier-demote-frequency", &freq);
if (ret) {
@@ -915,23 +1013,56 @@ tier_init (xlator_t *this)
defrag->tier_demote_frequency = freq;
- GF_OPTION_INIT ("write-freq-threshold",
- defrag->write_freq_threshold,
- int32, out);
+ ret = dict_get_int32 (this->options,
+ "write-freq-threshold", &freq);
+ if (ret) {
+ freq = DEFAULT_WRITE_FREQ_SEC;
+ }
+
+ defrag->write_freq_threshold = freq;
- GF_OPTION_INIT ("read-freq-threshold",
- defrag->read_freq_threshold,
- int32, out);
+ ret = dict_get_int32 (this->options,
+ "read-freq-threshold", &freq);
+ if (ret) {
+ freq = DEFAULT_READ_FREQ_SEC;
+ }
+
+ defrag->read_freq_threshold = freq;
+
+ ret = gf_asprintf(&promotion_qfile, "%s/%s-%d",
+ DEFAULT_VAR_RUN_DIRECTORY,
+ PROMOTION_QFILE,
+ getpid());
+ if (ret < 0)
+ goto out;
+
+ ret = gf_asprintf(&demotion_qfile, "%s/%s-%d",
+ DEFAULT_VAR_RUN_DIRECTORY,
+ DEMOTION_QFILE,
+ getpid());
+ if (ret < 0) {
+ GF_FREE(promotion_qfile);
+ goto out;
+ }
gf_msg(this->name, GF_LOG_INFO, 0,
DHT_MSG_LOG_TIER_STATUS,
- "Promote frequency set to %d demote set to %d",
+ "Promote/demote frequency %d/%d "
+ "Write/Read freq thresholds %d/%d",
defrag->tier_promote_frequency,
- defrag->tier_demote_frequency);
+ defrag->tier_demote_frequency,
+ defrag->write_freq_threshold,
+ defrag->read_freq_threshold);
+
+ gf_msg(this->name, GF_LOG_INFO, 0,
+ DHT_MSG_LOG_TIER_STATUS,
+ "Promote file %s demote file %s",
+ promotion_qfile, demotion_qfile);
ret = 0;
out:
+
return ret;
}
@@ -967,9 +1098,21 @@ out:
return dht_reconfigure (this, options);
}
+void
+tier_fini (xlator_t *this)
+{
+ if (libhandle)
+ dlclose(libhandle);
+
+ GF_FREE(demotion_qfile);
+ GF_FREE(promotion_qfile);
+
+ dht_fini(this);
+}
+
class_methods_t class_methods = {
.init = tier_init,
- .fini = dht_fini,
+ .fini = tier_fini,
.reconfigure = tier_reconfigure,
.notify = dht_notify
};
diff --git a/xlators/cluster/dht/src/tier.h b/xlators/cluster/dht/src/tier.h
index 70f50bfccae..6dc830dabfa 100644
--- a/xlators/cluster/dht/src/tier.h
+++ b/xlators/cluster/dht/src/tier.h
@@ -27,9 +27,11 @@
#define DEFAULT_PROMOTE_FREQ_SEC 120
#define DEFAULT_DEMOTE_FREQ_SEC 120
+#define DEFAULT_WRITE_FREQ_SEC 120
+#define DEFAULT_READ_FREQ_SEC 120
/*
- * Size of timer wheel. We would not promote or demote lesd
+ * Size of timer wheel. We would not promote or demote less
* frequently than this number.
*/
#define TIMER_SECS 3600
@@ -38,11 +40,11 @@
#include <ctype.h>
#include <sys/stat.h>
-#define DEMOTION_QFILE "/var/run/gluster/demotequeryfile"
-#define PROMOTION_QFILE "/var/run/gluster/promotequeryfile"
+#define PROMOTION_QFILE "promotequeryfile"
+#define DEMOTION_QFILE "demotequeryfile"
#define GET_QFILE_PATH(is_promotion)\
- (is_promotion) ? PROMOTION_QFILE : DEMOTION_QFILE
+ (is_promotion) ? promotion_qfile : demotion_qfile
typedef struct _query_cbk_args {
xlator_t *this;