summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--libglusterfs/src/dict.c2
-rw-r--r--libglusterfs/src/gfdb/Makefile.am8
-rw-r--r--libglusterfs/src/gfdb/gfdb_data_store.c11
-rw-r--r--libglusterfs/src/gfdb/gfdb_data_store.h16
-rw-r--r--libglusterfs/src/gfdb/gfdb_data_store_helper.c605
-rw-r--r--libglusterfs/src/gfdb/gfdb_data_store_helper.h125
-rw-r--r--libglusterfs/src/gfdb/gfdb_data_store_types.h179
-rw-r--r--libglusterfs/src/gfdb/gfdb_sqlite3.c174
-rw-r--r--libglusterfs/src/gfdb/gfdb_sqlite3.h1
-rw-r--r--libglusterfs/src/gfdb/gfdb_sqlite3_helper.c197
-rw-r--r--xlators/cluster/dht/src/tier.c340
-rw-r--r--xlators/cluster/dht/src/tier.h6
-rw-r--r--xlators/features/changetimerecorder/src/changetimerecorder.c25
-rw-r--r--xlators/features/changetimerecorder/src/ctr-helper.h2
14 files changed, 1207 insertions, 484 deletions
diff --git a/libglusterfs/src/dict.c b/libglusterfs/src/dict.c
index fc8a42cfc19..d9644ffeba8 100644
--- a/libglusterfs/src/dict.c
+++ b/libglusterfs/src/dict.c
@@ -2828,7 +2828,7 @@ dict_unserialize (char *orig_buf, int32_t size, dict_t **fill)
}
value = get_new_data ();
value->len = vallen;
-value->data = memdup (buf, vallen);
+ value->data = memdup (buf, vallen);
value->is_static = 0;
buf += vallen;
diff --git a/libglusterfs/src/gfdb/Makefile.am b/libglusterfs/src/gfdb/Makefile.am
index 30d1b7bcdde..50cf3402787 100644
--- a/libglusterfs/src/gfdb/Makefile.am
+++ b/libglusterfs/src/gfdb/Makefile.am
@@ -18,14 +18,14 @@ endif
CONTRIB_BUILDDIR = $(top_builddir)/contrib
-libgfdb_la_SOURCES = gfdb_data_store.c gfdb_sqlite3_helper.c\
+libgfdb_la_SOURCES = gfdb_data_store.c gfdb_data_store_helper.c gfdb_sqlite3_helper.c\
gfdb_sqlite3.c
noinst_HEADERS = gfdb_data_store.h gfdb_data_store_types.h gfdb_sqlite3_helper.h\
- gfdb_sqlite3.h gfdb_mem-types.h
+ gfdb_sqlite3.h gfdb_mem-types.h gfdb_data_store_helper.h
-libgfdb_HEADERS = gfdb_data_store.h gfdb_data_store_types.h \
- gfdb_sqlite3.h gfdb_mem-types.h gfdb_sqlite3_helper.c
+libgfdb_HEADERS = gfdb_data_store.h gfdb_data_store_types.h gfdb_data_store_helper.h\
+ gfdb_sqlite3.h gfdb_mem-types.h gfdb_sqlite3_helper.h
CLEANFILES =
diff --git a/libglusterfs/src/gfdb/gfdb_data_store.c b/libglusterfs/src/gfdb/gfdb_data_store.c
index e7ff815fc06..a9becd35807 100644
--- a/libglusterfs/src/gfdb/gfdb_data_store.c
+++ b/libglusterfs/src/gfdb/gfdb_data_store.c
@@ -797,4 +797,15 @@ void get_gfdb_methods (gfdb_methods_t *methods)
methods->get_db_version = get_db_version;
methods->get_db_setting = get_db_setting;
methods->get_db_path_key = get_db_path_key;
+
+ /* Query Record related functions */
+ methods->gfdb_query_record_new = gfdb_query_record_new;
+ methods->gfdb_query_record_free = gfdb_query_record_free;
+ methods->gfdb_add_link_to_query_record = gfdb_add_link_to_query_record;
+ methods->gfdb_write_query_record = gfdb_write_query_record;
+ methods->gfdb_read_query_record = gfdb_read_query_record;
+
+ /* Link info related functions */
+ methods->gfdb_link_info_new = gfdb_link_info_new;
+ methods->gfdb_link_info_free = gfdb_link_info_free;
}
diff --git a/libglusterfs/src/gfdb/gfdb_data_store.h b/libglusterfs/src/gfdb/gfdb_data_store.h
index 58a942c2d39..44fdef09b25 100644
--- a/libglusterfs/src/gfdb/gfdb_data_store.h
+++ b/libglusterfs/src/gfdb/gfdb_data_store.h
@@ -279,6 +279,8 @@ typedef int (*find_recently_changed_files_freq_t) (gfdb_conn_node_t *_conn_node,
gf_boolean_t _clear_counters);
+typedef const
+char *(*get_db_path_key_t)();
/*Libgfdb API Function: Clear the heat for all the files
*
@@ -349,7 +351,19 @@ typedef struct gfdb_methods_s {
/* Do not expose dbpath directly. Expose it via an */
/* access function: get_db_path_key(). */
char *dbpath;
- get_db_path_t get_db_path_key;
+ get_db_path_key_t get_db_path_key;
+
+ /* Query Record related functions */
+ gfdb_query_record_new_t gfdb_query_record_new;
+ gfdb_query_record_free_t gfdb_query_record_free;
+ gfdb_add_link_to_query_record_t gfdb_add_link_to_query_record;
+ gfdb_write_query_record_t gfdb_write_query_record;
+ gfdb_read_query_record_t gfdb_read_query_record;
+
+ /* Link info related functions */
+ gfdb_link_info_new_t gfdb_link_info_new;
+ gfdb_link_info_free_t gfdb_link_info_free;
+
} gfdb_methods_t;
void get_gfdb_methods (gfdb_methods_t *methods);
diff --git a/libglusterfs/src/gfdb/gfdb_data_store_helper.c b/libglusterfs/src/gfdb/gfdb_data_store_helper.c
new file mode 100644
index 00000000000..ff85e17169d
--- /dev/null
+++ b/libglusterfs/src/gfdb/gfdb_data_store_helper.c
@@ -0,0 +1,605 @@
+#include "gfdb_data_store_helper.h"
+
+
+/*Create a single link info structure*/
+gfdb_link_info_t*
+gfdb_link_info_new ()
+{
+ gfdb_link_info_t *link_info = NULL;
+
+ link_info = GF_CALLOC (1, sizeof(gfdb_link_info_t),
+ gf_mt_gfdb_link_info_t);
+ if (!link_info) {
+ gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, ENOMEM,
+ LG_MSG_NO_MEMORY, "Memory allocation failed for "
+ "link_info ");
+ goto out;
+ }
+
+ INIT_LIST_HEAD (&link_info->list);
+
+out:
+
+ return link_info;
+}
+
+/*Destroy a link info structure*/
+void
+gfdb_link_info_free(gfdb_link_info_t *link_info)
+{
+ GF_FREE (link_info);
+}
+
+
+/*Function to create the query_record*/
+gfdb_query_record_t *
+gfdb_query_record_new()
+{
+ int ret = -1;
+ gfdb_query_record_t *query_record = NULL;
+
+ query_record = GF_CALLOC (1, sizeof(gfdb_query_record_t),
+ gf_mt_gfdb_query_record_t);
+ if (!query_record) {
+ gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, ENOMEM,
+ LG_MSG_NO_MEMORY, "Memory allocation failed for "
+ "query_record ");
+ goto out;
+ }
+
+ INIT_LIST_HEAD (&query_record->link_list);
+
+ ret = 0;
+out:
+ if (ret == -1) {
+ GF_FREE (query_record);
+ }
+ return query_record;
+}
+
+
+/*Function to delete a single linkinfo from list*/
+static void
+gfdb_delete_linkinfo_from_list (gfdb_link_info_t **link_info)
+{
+ GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, link_info, out);
+ GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, *link_info, out);
+
+ /*Remove hard link from list*/
+ list_del(&(*link_info)->list);
+ gfdb_link_info_free (*link_info);
+ link_info = NULL;
+out:
+ return;
+}
+
+
+/*Function to destroy link_info list*/
+void
+gfdb_free_link_info_list (gfdb_query_record_t *query_record)
+{
+ gfdb_link_info_t *link_info = NULL;
+ gfdb_link_info_t *temp = NULL;
+
+ GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, query_record, out);
+
+ list_for_each_entry_safe(link_info, temp,
+ &query_record->link_list, list)
+ {
+ gfdb_delete_linkinfo_from_list (&link_info);
+ link_info = NULL;
+ }
+
+out:
+ return;
+}
+
+
+
+/* Function to add linkinfo to the query record */
+int
+gfdb_add_link_to_query_record (gfdb_query_record_t *query_record,
+ uuid_t pgfid,
+ char *base_name)
+{
+ int ret = -1;
+ gfdb_link_info_t *link_info = NULL;
+ int base_name_len = 0;
+
+ GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, query_record, out);
+ GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, pgfid, out);
+ GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, base_name, out);
+
+ link_info = gfdb_link_info_new ();
+ if (!link_info) {
+ goto out;
+ }
+
+ gf_uuid_copy (link_info->pargfid, pgfid);
+ base_name_len = strlen (base_name);
+ memcpy (link_info->file_name, base_name, base_name_len);
+ link_info->file_name[base_name_len] = '\0';
+
+ list_add_tail (&link_info->list,
+ &query_record->link_list);
+
+ query_record->link_count++;
+
+ ret = 0;
+out:
+ if (ret) {
+ gfdb_link_info_free (link_info);
+ link_info = NULL;
+ }
+ return ret;
+}
+
+
+
+/*Function to destroy query record*/
+void
+gfdb_query_record_free(gfdb_query_record_t *query_record)
+{
+ if (query_record) {
+ gfdb_free_link_info_list (query_record);
+ GF_FREE (query_record);
+ }
+}
+
+
+/******************************************************************************
+ SERIALIZATION/DE-SERIALIZATION OF QUERY RECORD
+*******************************************************************************/
+/******************************************************************************
+ The on disk format of query record is as follows,
+
++---------------------------------------------------------------------------+
+| Length of serialized query record | Serialized Query Record |
++---------------------------------------------------------------------------+
+ 4 bytes Length of serialized query record
+ |
+ |
+ -------------------------------------------------|
+ |
+ |
+ V
+ Serialized Query Record Format:
+ +---------------------------------------------------------------------------+
+ | GFID | Link count | <LINK INFO> |..... | FOOTER |
+ +---------------------------------------------------------------------------+
+ 16 B 4 B Link Length 4 B
+ | |
+ | |
+ -----------------------------| |
+ | |
+ | |
+ V |
+ Each <Link Info> will be serialized as |
+ +-----------------------------------------------+ |
+ | PGID | BASE_NAME_LENGTH | BASE_NAME | |
+ +-----------------------------------------------+ |
+ 16 B 4 B BASE_NAME_LENGTH |
+ |
+ |
+ ------------------------------------------------------------------------|
+ |
+ |
+ V
+ FOOTER is a magic number 0xBAADF00D indicating the end of the record.
+ This also serves as a serialized schema validator.
+ * ****************************************************************************/
+
+#define GFDB_QUERY_RECORD_FOOTER 0xBAADF00D
+#define UUID_LEN 16
+
+/*Function to get the potential length of the serialized buffer*/
+static int32_t
+gfdb_query_record_serialized_length (gfdb_query_record_t *query_record)
+{
+ int32_t len = -1;
+ gfdb_link_info_t *link_info = NULL;
+
+ GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, query_record, out);
+
+ /* Length of GFID */
+ len = UUID_LEN;
+
+ /* length of number of links*/
+ len += sizeof (int32_t);
+
+ list_for_each_entry (link_info, &query_record->link_list, list) {
+
+ /* length of PFID */
+ len += UUID_LEN;
+
+ /* Add size of base name length*/
+ len += sizeof (int32_t);
+
+ /* Length of base_name */
+ len += strlen (link_info->file_name);
+
+ }
+
+ /* length of footer */
+ len += sizeof (int32_t);
+out:
+ return len;
+}
+
+/* Function for serializing query record.
+ *
+ * Query Record Serialization Format
+ * +---------------------------------------------------------------------------+
+ * | GFID | Link count | <LINK INFO> |..... | FOOTER |
+ * +---------------------------------------------------------------------------+
+ * 16 B 4 B Link Length 4 B
+ *
+ *
+ * Each <Link Info> will be serialized as
+ * +-----------------------------------------------+
+ * | PGID | BASE_NAME_LENGTH | BASE_NAME |
+ * +-----------------------------------------------+
+ * 16 B 4 B BASE_NAME_LENGTH
+ *
+ *
+ * FOOTER is a magic number 0xBAADF00D indicating the end of the record.
+ * This also serves as a serialized schema validator.
+ *
+ * The function will allocate memory to the serialized buffer,
+ * the caller needs to free it.
+ * Returns the length of the serialized buffer on success
+ * or -1 on failure.
+ *
+ * */
+static int
+gfdb_query_record_serialize (gfdb_query_record_t *query_record,
+ char **in_buffer)
+{
+ gfdb_link_info_t *link_info = NULL;
+ int count = -1;
+ int base_name_len = 0;
+ int buffer_length = 0;
+ int footer = GFDB_QUERY_RECORD_FOOTER;
+ char *buffer = NULL;
+ char *ret_buffer = NULL;
+
+ GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, query_record, out);
+ GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE,
+ (query_record->link_count > 0), out);
+ GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, in_buffer, out);
+
+
+ /* Calculate the total length of the serialized buffer */
+ buffer_length = gfdb_query_record_serialized_length (query_record);
+ if (buffer_length <= 0) {
+ gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
+ LG_MSG_DB_ERROR, "Failed to calculate the length of "
+ "serialized buffer");
+ goto out;
+ }
+
+ /* Allocate memory to the serialized buffer */
+ ret_buffer = GF_CALLOC (1, buffer_length, gf_common_mt_char);
+ if (!ret_buffer) {
+ gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
+ LG_MSG_DB_ERROR, "Memory allocation failed for "
+ "serialized buffer.");
+ goto out;
+ }
+
+ buffer = ret_buffer;
+
+ count = 0;
+
+ /* Copying the GFID */
+ memcpy (buffer, query_record->gfid, UUID_LEN);
+ buffer += UUID_LEN;
+ count += UUID_LEN;
+
+ /* Copying the number of links */
+ memcpy (buffer, &query_record->link_count, sizeof (int32_t));
+ buffer += sizeof (int32_t);
+ count += sizeof (int32_t);
+
+ list_for_each_entry (link_info, &query_record->link_list, list) {
+
+ /* Copying the PFID */
+ memcpy(buffer, link_info->pargfid, UUID_LEN);
+ buffer += UUID_LEN;
+ count += UUID_LEN;
+
+ /* Copying base name length*/
+ base_name_len = strlen (link_info->file_name);
+ memcpy (buffer, &base_name_len, sizeof (int32_t));
+ buffer += sizeof (int32_t);
+ count += sizeof (int32_t);
+
+ /* Length of base_name */
+ memcpy(buffer, link_info->file_name, base_name_len);
+ buffer += base_name_len;
+ count += base_name_len;
+
+ }
+
+ /* Copying the Footer of the record */
+ memcpy (buffer, &footer, sizeof (int32_t));
+ buffer += sizeof (int32_t);
+ count += sizeof (int32_t);
+
+out:
+ if (count < 0) {
+ GF_FREE (ret_buffer);
+ ret_buffer = NULL;
+ }
+ *in_buffer = ret_buffer;
+ return count;
+}
+
+static gf_boolean_t
+is_serialized_buffer_valid (char *in_buffer, int buffer_length) {
+ gf_boolean_t ret = _gf_false;
+ int footer = 0;
+
+ /* Read the footer */
+ in_buffer += (buffer_length - sizeof (int32_t));
+ memcpy (&footer, in_buffer, sizeof (int32_t));
+
+ /*
+ * if the footer is not GFDB_QUERY_RECORD_FOOTER
+ * then the serialized record is invalid
+ *
+ * */
+ if (footer != GFDB_QUERY_RECORD_FOOTER) {
+ goto out;
+ }
+
+ ret = _gf_true;
+out:
+ return ret;
+}
+
+
+static int
+gfdb_query_record_deserialize (char *in_buffer,
+ int buffer_length,
+ gfdb_query_record_t **query_record)
+{
+ int ret = -1;
+ char *buffer = NULL;
+ int i = 0;
+ gfdb_link_info_t *link_info = NULL;
+ int count = 0;
+ int base_name_len = 0;
+ gfdb_query_record_t *ret_qrecord = NULL;
+
+ GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, in_buffer, out);
+ GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, query_record, out);
+ GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, (buffer_length > 0), out);
+
+ if (!is_serialized_buffer_valid (in_buffer, buffer_length)) {
+ gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
+ LG_MSG_DB_ERROR, "Invalid serialized query record");
+ goto out;
+ }
+
+ buffer = in_buffer;
+
+ ret_qrecord = gfdb_query_record_new ();
+ if (!ret_qrecord) {
+ gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
+ LG_MSG_DB_ERROR, "Failed to allocate space to "
+ "gfdb_query_record_t");
+ goto out;
+ }
+
+ /* READ GFID */
+ memcpy ((ret_qrecord)->gfid, buffer, UUID_LEN);
+ buffer += UUID_LEN;
+ count += UUID_LEN;
+
+ /* Read the number of link */
+ memcpy (&(ret_qrecord->link_count), buffer, sizeof (int32_t));
+ buffer += sizeof (int32_t);
+ count += sizeof (int32_t);
+
+ /* Read all the links */
+ for (i = 0; i < ret_qrecord->link_count; i++) {
+ if (count >= buffer_length) {
+ gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
+ LG_MSG_DB_ERROR, "Invalid serialized "
+ "query record");
+ ret = -1;
+ goto out;
+ }
+
+ link_info = gfdb_link_info_new ();
+ if (!link_info) {
+ gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
+ LG_MSG_DB_ERROR, "Failed to create link_info");
+ goto out;
+ }
+
+ /* READ PGFID */
+ memcpy (link_info->pargfid, buffer, UUID_LEN);
+ buffer += UUID_LEN;
+ count += UUID_LEN;
+
+ /* Read base name length */
+ memcpy (&base_name_len, buffer, sizeof (int32_t));
+ buffer += sizeof (int32_t);
+ count += sizeof (int32_t);
+
+ /* READ basename */
+ memcpy (link_info->file_name, buffer, base_name_len);
+ buffer += base_name_len;
+ count += base_name_len;
+ link_info->file_name[base_name_len] = '\0';
+
+ /* Add link_info to the list */
+ list_add_tail (&link_info->list,
+ &(ret_qrecord->link_list));
+
+ /* Reseting link_info */
+ link_info = NULL;
+ }
+
+ ret = 0;
+out:
+ if (ret) {
+ gfdb_query_record_free (ret_qrecord);
+ ret_qrecord = NULL;
+ }
+ *query_record = ret_qrecord;
+ return ret;
+}
+
+
+
+
+
+/* Function to write query record to file
+ *
+ * Disk format
+ * +---------------------------------------------------------------------------+
+ * | Length of serialized query record | Serialized Query Record |
+ * +---------------------------------------------------------------------------+
+ * 4 bytes Length of serialized query record
+ *
+ * Please refer gfdb_query_record_serialize () for format of
+ * Serialized Query Record
+ *
+ * */
+int
+gfdb_write_query_record (int fd,
+ gfdb_query_record_t *query_record)
+{
+ int ret = -1;
+ int buffer_len = 0;
+ char *buffer = NULL;
+ int write_len = 0;
+ char *write_buffer = NULL;
+
+ GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, (fd >= 0), out);
+ GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, query_record, out);
+
+ buffer_len = gfdb_query_record_serialize (query_record, &buffer);
+ if (buffer_len < 0) {
+ gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
+ LG_MSG_DB_ERROR, "Failed to serialize query record");
+ goto out;
+ }
+
+ /* Serialize the buffer length and write to file */
+ ret = write (fd, &buffer_len, sizeof (int32_t));
+ if (ret < 0) {
+ gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
+ LG_MSG_DB_ERROR, "Failed to write buffer length"
+ " to file");
+ goto out;
+ }
+
+ /* Write the serialized query record to file */
+ write_len = buffer_len;
+ write_buffer = buffer;
+ while ((ret = write (fd, write_buffer, write_len)) < write_len) {
+ if (ret < 0) {
+ gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, errno,
+ LG_MSG_DB_ERROR, "Failed to write serialized "
+ "query record to file");
+ goto out;
+ }
+
+ write_buffer += ret;
+ write_len -= ret;
+ }
+
+ ret = 0;
+out:
+ GF_FREE (buffer);
+ return ret;
+}
+
+
+
+/* Function to read query record from file.
+ * Allocates memory to query record and
+ * returns length of serialized query record when successful
+ * Return -1 when failed.
+ * Return 0 when reached EOF.
+ * */
+int
+gfdb_read_query_record (int fd,
+ gfdb_query_record_t **query_record)
+{
+ int ret = -1;
+ int buffer_len = 0;
+ int read_len = 0;
+ char *buffer = NULL;
+ char *read_buffer = NULL;
+
+ GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, (fd >= 0), out);
+ GF_VALIDATE_OR_GOTO (GFDB_DATA_STORE, query_record, out);
+
+
+ /* Read serialized query record length from the file*/
+ ret = read (fd, &buffer_len, sizeof (int32_t));
+ if (ret < 0) {
+ gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
+ LG_MSG_DB_ERROR, "Failed reading buffer length"
+ " from file");
+ goto out;
+ }
+ /* EOF */
+ else if (ret == 0) {
+ ret = 0;
+ goto out;
+ }
+
+ /* Allocating memory to the serialization buffer */
+ buffer = GF_CALLOC (1, buffer_len, gf_common_mt_char);
+ if (!buffer) {
+ gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
+ LG_MSG_DB_ERROR, "Failed to allocate space to "
+ "serialized buffer");
+ goto out;
+ }
+
+
+ /* Read the serialized query record from file */
+ read_len = buffer_len;
+ read_buffer = buffer;
+ while ((ret = read (fd, read_buffer, read_len)) < read_len) {
+
+ /*Any error */
+ if (ret < 0) {
+ gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, errno,
+ LG_MSG_DB_ERROR, "Failed to read serialized "
+ "query record from file");
+ goto out;
+ }
+ /* EOF */
+ else if (ret == 0) {
+ gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
+ LG_MSG_DB_ERROR, "Invalid query record or "
+ "corrupted query file");
+ ret = -1;
+ goto out;
+ }
+
+ read_buffer += ret;
+ read_len -= ret;
+ }
+
+ ret = gfdb_query_record_deserialize (buffer, buffer_len,
+ query_record);
+ if (ret) {
+ gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, 0,
+ LG_MSG_DB_ERROR, "Failed to de-serialize query record");
+ goto out;
+ }
+
+ ret = buffer_len;
+out:
+ GF_FREE (buffer);
+ return ret;
+}
diff --git a/libglusterfs/src/gfdb/gfdb_data_store_helper.h b/libglusterfs/src/gfdb/gfdb_data_store_helper.h
new file mode 100644
index 00000000000..fe9fbba8795
--- /dev/null
+++ b/libglusterfs/src/gfdb/gfdb_data_store_helper.h
@@ -0,0 +1,125 @@
+/*
+ Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+*/
+#ifndef __GFDB_DATA_STORE_HELPER_H
+#define __GFDB_DATA_STORE_HELPER_H
+
+#include <time.h>
+#include <sys/time.h>
+#include <string.h>
+
+#include "common-utils.h"
+#include "compat-uuid.h"
+#include "gfdb_mem-types.h"
+#include "dict.h"
+#include "byte-order.h"
+#include "libglusterfs-messages.h"
+
+
+#define GFDB_DATA_STORE "gfdbdatastore"
+
+/*******************************************************************************
+ *
+ * Query related data structure and functions
+ *
+ * ****************************************************************************/
+
+#ifdef NAME_MAX
+#define GF_NAME_MAX NAME_MAX
+#else
+#define GF_NAME_MAX 255
+#endif
+
+/*Structure to hold the link information*/
+typedef struct gfdb_link_info {
+ uuid_t pargfid;
+ char file_name[GF_NAME_MAX];
+ struct list_head list;
+} gfdb_link_info_t;
+
+
+/*Create a single link info structure*/
+gfdb_link_info_t *gfdb_link_info_new ();
+typedef gfdb_link_info_t *(*gfdb_link_info_new_t) ();
+
+/*Destroy a link info structure*/
+void
+gfdb_link_info_free (gfdb_link_info_t *gfdb_link_info);
+typedef void
+(*gfdb_link_info_free_t) (gfdb_link_info_t *gfdb_link_info);
+
+
+
+
+
+/*Structure used for querying purpose*/
+typedef struct gfdb_query_record {
+ uuid_t gfid;
+ /*This is the hardlink list*/
+ struct list_head link_list;
+ int link_count;
+} gfdb_query_record_t;
+
+
+
+/* Function to create the query_record */
+gfdb_query_record_t *
+gfdb_query_record_new();
+typedef gfdb_query_record_t *
+(*gfdb_query_record_new_t)();
+
+
+
+
+/* Fuction to add linkinfo to query record */
+int
+gfdb_add_link_to_query_record (gfdb_query_record_t *gfdb_query_record,
+ uuid_t pgfid,
+ char *base_name);
+typedef int
+(*gfdb_add_link_to_query_record_t) (gfdb_query_record_t *, uuid_t, char *);
+
+
+
+
+/*Function to destroy query record*/
+void
+gfdb_query_record_free (gfdb_query_record_t *gfdb_query_record);
+typedef void
+(*gfdb_query_record_free_t) (gfdb_query_record_t *);
+
+
+
+
+
+
+/* Function to write query record to file */
+int
+gfdb_write_query_record (int fd,
+ gfdb_query_record_t *gfdb_query_record);
+typedef int
+(*gfdb_write_query_record_t) (int, gfdb_query_record_t *);
+
+
+
+
+
+/* Function to read query record from file.
+ * Allocates memory to query record and return 0 when successful
+ * Return -1 when failed.
+ * Return 0 when EOF.
+ * */
+int
+gfdb_read_query_record (int fd,
+ gfdb_query_record_t **gfdb_query_record);
+typedef int
+(*gfdb_read_query_record_t) (int, gfdb_query_record_t **);
+
+
+#endif \ No newline at end of file
diff --git a/libglusterfs/src/gfdb/gfdb_data_store_types.h b/libglusterfs/src/gfdb/gfdb_data_store_types.h
index 4ad2bd4feb7..ce09e731746 100644
--- a/libglusterfs/src/gfdb/gfdb_data_store_types.h
+++ b/libglusterfs/src/gfdb/gfdb_data_store_types.h
@@ -10,21 +10,7 @@
#ifndef __GFDB_DATA_STORE_TYPE_H
#define __GFDB_DATA_STORE_TYPE_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <time.h>
-#include <sys/time.h>
-#include <string.h>
-
-#include "common-utils.h"
-#include "compat-uuid.h"
-#include "gfdb_mem-types.h"
-#include "dict.h"
-#include "libglusterfs-messages.h"
+#include "gfdb_data_store_helper.h"
/*
* Helps in dynamically choosing log level
@@ -154,7 +140,6 @@ typedef enum gfdb_db_type {
} gfdb_db_type_t;
/*String related to the db types*/
-#define GFDB_DATA_STORE "gfdbdatastore"
#define GFDB_STR_HASH_FILE_STORE "hashfile"
#define GFDB_STR_ROCKS_DB "rocksdb"
#define GFDB_STR_SQLITE3 "sqlite3"
@@ -319,168 +304,6 @@ typedef struct gfdb_db_record {
} gfdb_db_record_t;
-
-/*******************************************************************************
- *
- * Query related data structure and functions
- *
- * ****************************************************************************/
-
-
-
-/*Structure used for querying purpose*/
-typedef struct gfdb_query_record {
- /*Inode info*/
- uuid_t gfid;
- /*All the hard link of the inode
- * All the hard links will be queried as
- * "GF_PID,FNAME,FPATH,W_DEL_FLAG,LINK_UPDATE"
- * and multiple hardlinks will be seperated by "::"*/
- /*Do only shallow copy. The gf_query_callback_t */
- /* function should do the deep copy.*/
- char *_link_info_str;
- ssize_t link_info_size;
-} gfdb_query_record_t;
-
-/*Function to create the query_record*/
-static inline gfdb_query_record_t *
-gfdb_query_record_init()
-{
- int ret = -1;
- gfdb_query_record_t *gfdb_query_record = NULL;
-
- gfdb_query_record = GF_CALLOC (1, sizeof(gfdb_query_record_t),
- gf_mt_gfdb_query_record_t);
- if (!gfdb_query_record) {
- gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, ENOMEM,
- LG_MSG_NO_MEMORY, "Error allocating memory to "
- "gfdb_query_record ");
- goto out;
- }
- ret = 0;
-out:
- if (ret == -1) {
- GF_FREE (gfdb_query_record);
- }
- return gfdb_query_record;
-}
-
-/*Function to destroy query record*/
-static inline void
-gfdb_query_record_fini(gfdb_query_record_t
- **gfdb_query_record) {
- GF_FREE (*gfdb_query_record);
-}
-
-
-
-
-
-
-
-
-/*Structure to hold the link information*/
-typedef struct gfdb_link_info {
- uuid_t pargfid;
- char file_name[PATH_MAX];
- char file_path[PATH_MAX];
- gf_boolean_t is_link_updated;
- gf_boolean_t is_del_flag_set;
-} gfdb_link_info_t;
-
-/*Create a single link info structure*/
-static inline gfdb_link_info_t *
-gfdb_link_info_init ()
-{
- gfdb_link_info_t *gfdb_link_info = NULL;
-
- gfdb_link_info = GF_CALLOC (1, sizeof(gfdb_link_info_t),
- gf_mt_gfdb_link_info_t);
- if (!gfdb_link_info) {
- gf_msg (GFDB_DATA_STORE, GF_LOG_ERROR, ENOMEM,
- LG_MSG_NO_MEMORY, "Error allocating memory to "
- "gfdb_link_info ");
- }
-
- return gfdb_link_info;
-}
-
-/*Destroy a link info structure*/
-static inline void
-gfdb_link_info_fini(gfdb_link_info_t **gfdb_link_info)
-{
- if (gfdb_link_info)
- GF_FREE (*gfdb_link_info);
-}
-
-
-/*Length of each hard link string */
-#define DEFAULT_LINK_INFO_STR_LEN 1024
-
-/* Parse a single link string into link_info structure
- * Input format of str_link
- * "GF_PID,FNAME,FPATH,W_DEL_FLAG,LINK_UPDATE"
- *
- * */
-static inline int
-str_to_link_info (char *str_link,
- gfdb_link_info_t *link_info)
-{
- int ret = -1;
- const char *delimiter = ",";
- char *token_str = NULL;
- char *saveptr = NULL;
- char gfid[200] = "";
-
- GF_ASSERT (str_link);
- GF_ASSERT (link_info);
-
- /*Parent GFID*/
- token_str = strtok_r(str_link, delimiter, &saveptr);
- if (token_str != NULL) {
- strcpy (gfid, token_str);
- ret = gf_uuid_parse (gfid, link_info->pargfid);
- if (ret == -1)
- goto out;
- }
-
- /*Filename*/
- token_str = strtok_r(NULL, delimiter, &saveptr);
- if (token_str != NULL) {
- strcpy (link_info->file_name, token_str);
- }
-
- /*Filepath*/
- token_str = strtok_r(NULL, delimiter, &saveptr);
- if (token_str != NULL) {
- strcpy (link_info->file_path, token_str);
- }
-
- /*is_link_updated*/
- token_str = strtok_r(NULL, delimiter, &saveptr);
- if (token_str != NULL) {
- link_info->is_link_updated = atoi(token_str);
- if (link_info->is_link_updated != 0 &&
- link_info->is_link_updated != 1) {
- goto out;
- }
- }
-
- /*is_del_flag_set*/
- token_str = strtok_r(NULL, delimiter, &saveptr);
- if (token_str != NULL) {
- link_info->is_del_flag_set = atoi (token_str);
- if (link_info->is_del_flag_set != 0 &&
- link_info->is_del_flag_set != 1) {
- goto out;
- }
- }
- ret = 0;
-out:
- return ret;
-}
-
-
/*******************************************************************************
*
* Signatures for the plugin functions
diff --git a/libglusterfs/src/gfdb/gfdb_sqlite3.c b/libglusterfs/src/gfdb/gfdb_sqlite3.c
index 642aa76697d..d41c9ab105b 100644
--- a/libglusterfs/src/gfdb/gfdb_sqlite3.c
+++ b/libglusterfs/src/gfdb/gfdb_sqlite3.c
@@ -604,6 +604,28 @@ out:
*
* ***************************************************************************/
+static int
+gf_get_basic_query_stmt (char **out_stmt)
+{
+ int ret = -1;
+ ret = gf_asprintf (out_stmt, "select GF_FILE_TB.GF_ID,"
+ "GF_FLINK_TB.GF_PID ,"
+ "GF_FLINK_TB.FNAME "
+ "from GF_FLINK_TB, GF_FILE_TB "
+ "where "
+ "GF_FILE_TB.GF_ID = GF_FLINK_TB.GF_ID ");
+ if (ret <= 0) {
+ gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_QUERY_FAILED,
+ "Failed to create base query statement");
+ *out_stmt = NULL;
+ }
+ return ret;
+}
+
+
+
+
+
/*
* Find All files recorded in the DB
* Input:
@@ -619,22 +641,19 @@ gf_sqlite3_find_all (void *db_conn, gf_query_callback_t query_callback,
gf_sql_connection_t *sql_conn = db_conn;
sqlite3_stmt *prep_stmt = NULL;
-
CHECK_SQL_CONN (sql_conn, out);
GF_VALIDATE_OR_GOTO(GFDB_STR_SQLITE3, query_callback, out);
- query_str = "select GF_FILE_TB.GF_ID,"
- " (select group_concat( GF_PID || ',' || FNAME || ','"
- " || FPATH || ',' || W_DEL_FLAG ||',' || LINK_UPDATE , '::')"
- " from GF_FLINK_TB where "
- "GF_FILE_TB.GF_ID = GF_FLINK_TB.GF_ID) from GF_FILE_TB ;";
-
+ ret = gf_get_basic_query_stmt (&query_str);
+ if (ret <= 0) {
+ goto out;
+ }
ret = sqlite3_prepare (sql_conn->sqlite3_db_conn, query_str, -1,
&prep_stmt, 0);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_PREPARE_FAILED, "Failed preparing statment %s :"
+ LG_MSG_PREPARE_FAILED, "Failed to prepare statment %s :"
"%s", query_str,
sqlite3_errmsg (sql_conn->sqlite3_db_conn));
ret = -1;
@@ -651,6 +670,7 @@ gf_sqlite3_find_all (void *db_conn, gf_query_callback_t query_callback,
ret = 0;
out:
sqlite3_finalize (prep_stmt);
+ GF_FREE (query_str);
return ret;
}
@@ -673,22 +693,31 @@ gf_sqlite3_find_recently_changed_files(void *db_conn,
gf_sql_connection_t *sql_conn = db_conn;
sqlite3_stmt *prep_stmt = NULL;
uint64_t from_time_usec = 0;
+ char *base_query_str = NULL;
CHECK_SQL_CONN (sql_conn, out);
GF_VALIDATE_OR_GOTO(GFDB_STR_SQLITE3, query_callback, out);
- query_str = "select GF_FILE_TB.GF_ID,"
- " (select group_concat( GF_PID || ',' || FNAME || ','"
- " || FPATH || ',' || W_DEL_FLAG ||',' || LINK_UPDATE , '::')"
- " from GF_FLINK_TB where GF_FILE_TB.GF_ID = GF_FLINK_TB.GF_ID)"
- " from GF_FILE_TB where "
+ ret = gf_get_basic_query_stmt (&base_query_str);
+ if (ret <= 0) {
+ goto out;
+ }
+
+ ret = gf_asprintf (&query_str, "%s AND"
/*First condition: For writes*/
"((" GF_COL_TB_WSEC " * " TOSTRING(GFDB_MICROSEC) " + "
GF_COL_TB_WMSEC ") >= ? )"
" OR "
/*Second condition: For reads*/
"((" GF_COL_TB_RWSEC " * " TOSTRING(GFDB_MICROSEC) " + "
- GF_COL_TB_RWMSEC ") >= ?)";
+ GF_COL_TB_RWMSEC ") >= ?)", base_query_str);
+
+ if (ret < 0) {
+ gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_QUERY_FAILED,
+ "Failed creating query statement");
+ query_str = NULL;
+ goto out;
+ }
from_time_usec = gfdb_time_2_usec (from_time);
@@ -696,7 +725,7 @@ gf_sqlite3_find_recently_changed_files(void *db_conn,
&prep_stmt, 0);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_PREPARE_FAILED, "Failed preparing statment %s :"
+ LG_MSG_PREPARE_FAILED, "Failed to prepare statment %s :"
" %s", query_str,
sqlite3_errmsg (sql_conn->sqlite3_db_conn));
ret = -1;
@@ -707,7 +736,7 @@ gf_sqlite3_find_recently_changed_files(void *db_conn,
ret = sqlite3_bind_int64 (prep_stmt, 1, from_time_usec);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_BINDING_FAILED, "Failed binding from_time_usec "
+ LG_MSG_BINDING_FAILED, "Failed to bind from_time_usec "
"%"PRIu64" : %s", from_time_usec,
sqlite3_errmsg (sql_conn->sqlite3_db_conn));
ret = -1;
@@ -718,7 +747,7 @@ gf_sqlite3_find_recently_changed_files(void *db_conn,
ret = sqlite3_bind_int64 (prep_stmt, 2, from_time_usec);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_BINDING_FAILED, "Failed binding from_time_usec "
+ LG_MSG_BINDING_FAILED, "Failed to bind from_time_usec "
"%"PRIu64" : %s ", from_time_usec,
sqlite3_errmsg (sql_conn->sqlite3_db_conn));
ret = -1;
@@ -736,6 +765,8 @@ gf_sqlite3_find_recently_changed_files(void *db_conn,
ret = 0;
out:
sqlite3_finalize (prep_stmt);
+ GF_FREE (base_query_str);
+ GF_FREE (query_str);
return ret;
}
@@ -757,23 +788,32 @@ gf_sqlite3_find_unchanged_for_time (void *db_conn,
char *query_str = NULL;
gf_sql_connection_t *sql_conn = db_conn;
sqlite3_stmt *prep_stmt = NULL;
- uint64_t for_time_usec = 0;
+ uint64_t for_time_usec = 0;
+ char *base_query_str = NULL;
CHECK_SQL_CONN (sql_conn, out);
GF_VALIDATE_OR_GOTO(GFDB_STR_SQLITE3, query_callback, out);
- query_str = "select GF_FILE_TB.GF_ID,"
- " (select group_concat( GF_PID || ',' || FNAME || ','"
- " || FPATH || ',' || W_DEL_FLAG ||',' || LINK_UPDATE , '::')"
- " from GF_FLINK_TB where GF_FILE_TB.GF_ID = GF_FLINK_TB.GF_ID)"
- " from GF_FILE_TB where "
+ ret = gf_get_basic_query_stmt (&base_query_str);
+ if (ret <= 0) {
+ goto out;
+ }
+
+ ret = gf_asprintf (&query_str, "%s AND "
/*First condition: For writes*/
"((" GF_COL_TB_WSEC " * " TOSTRING(GFDB_MICROSEC) " + "
GF_COL_TB_WMSEC ") <= ? )"
" AND "
/*Second condition: For reads*/
"((" GF_COL_TB_RWSEC " * " TOSTRING(GFDB_MICROSEC) " + "
- GF_COL_TB_RWMSEC ") <= ?)";
+ GF_COL_TB_RWMSEC ") <= ?)", base_query_str);
+
+ if (ret < 0) {
+ gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_QUERY_FAILED,
+ "Failed to create query statement");
+ query_str = NULL;
+ goto out;
+ }
for_time_usec = gfdb_time_2_usec (for_time);
@@ -781,7 +821,7 @@ gf_sqlite3_find_unchanged_for_time (void *db_conn,
&prep_stmt, 0);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_PREPARE_FAILED, "Failed preparing statment %s :"
+ LG_MSG_PREPARE_FAILED, "Failed to prepare statment %s :"
" %s", query_str,
sqlite3_errmsg (sql_conn->sqlite3_db_conn));
ret = -1;
@@ -792,7 +832,7 @@ gf_sqlite3_find_unchanged_for_time (void *db_conn,
ret = sqlite3_bind_int64 (prep_stmt, 1, for_time_usec);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_BINDING_FAILED, "Failed binding for_time_usec "
+ LG_MSG_BINDING_FAILED, "Failed to bind for_time_usec "
"%"PRIu64" : %s", for_time_usec,
sqlite3_errmsg (sql_conn->sqlite3_db_conn));
ret = -1;
@@ -803,7 +843,7 @@ gf_sqlite3_find_unchanged_for_time (void *db_conn,
ret = sqlite3_bind_int64 (prep_stmt, 2, for_time_usec);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_BINDING_FAILED, "Failed binding for_time_usec "
+ LG_MSG_BINDING_FAILED, "Failed to bind for_time_usec "
"%"PRIu64" : %s", for_time_usec,
sqlite3_errmsg (sql_conn->sqlite3_db_conn));
ret = -1;
@@ -821,6 +861,8 @@ gf_sqlite3_find_unchanged_for_time (void *db_conn,
ret = 0;
out:
sqlite3_finalize (prep_stmt);
+ GF_FREE (base_query_str);
+ GF_FREE (query_str);
return ret;
}
@@ -853,15 +895,16 @@ gf_sqlite3_find_recently_changed_files_freq (void *db_conn,
gf_sql_connection_t *sql_conn = db_conn;
sqlite3_stmt *prep_stmt = NULL;
uint64_t from_time_usec = 0;
+ char *base_query_str = NULL;
CHECK_SQL_CONN (sql_conn, out);
GF_VALIDATE_OR_GOTO(GFDB_STR_SQLITE3, query_callback, out);
- query_str = "select GF_FILE_TB.GF_ID,"
- " (select group_concat( GF_PID || ',' || FNAME || ','"
- " || FPATH || ',' || W_DEL_FLAG ||',' || LINK_UPDATE , '::')"
- " from GF_FLINK_TB where GF_FILE_TB.GF_ID = GF_FLINK_TB.GF_ID)"
- " from GF_FILE_TB where "
+ ret = gf_get_basic_query_stmt (&base_query_str);
+ if (ret <= 0) {
+ goto out;
+ }
+ ret = gf_asprintf (&query_str, "%s AND "
/*First condition: For Writes*/
"( ((" GF_COL_TB_WSEC " * " TOSTRING(GFDB_MICROSEC) " + "
GF_COL_TB_WMSEC ") >= ? )"
@@ -870,7 +913,14 @@ gf_sqlite3_find_recently_changed_files_freq (void *db_conn,
/*Second condition: For Reads */
"( ((" GF_COL_TB_RWSEC " * " TOSTRING(GFDB_MICROSEC) " + "
GF_COL_TB_RWMSEC ") >= ?)"
- " AND "" (" GF_COL_TB_RFC " >= ? ) )";
+ " AND "" (" GF_COL_TB_RFC " >= ? ) )", base_query_str);
+
+ if (ret < 0) {
+ gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_QUERY_FAILED,
+ "Failed to create query statement");
+ query_str = NULL;
+ goto out;
+ }
from_time_usec = gfdb_time_2_usec (from_time);
@@ -878,7 +928,7 @@ gf_sqlite3_find_recently_changed_files_freq (void *db_conn,
&prep_stmt, 0);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_PREPARE_FAILED, "Failed preparing statment %s :"
+ LG_MSG_PREPARE_FAILED, "Failed to prepare statment %s :"
" %s", query_str,
sqlite3_errmsg (sql_conn->sqlite3_db_conn));
ret = -1;
@@ -889,7 +939,7 @@ gf_sqlite3_find_recently_changed_files_freq (void *db_conn,
ret = sqlite3_bind_int64 (prep_stmt, 1, from_time_usec);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_BINDING_FAILED, "Failed binding from_time_usec "
+ LG_MSG_BINDING_FAILED, "Failed to bind from_time_usec "
"%"PRIu64" : %s", from_time_usec,
sqlite3_errmsg (sql_conn->sqlite3_db_conn));
ret = -1;
@@ -900,7 +950,7 @@ gf_sqlite3_find_recently_changed_files_freq (void *db_conn,
ret = sqlite3_bind_int (prep_stmt, 2, freq_write_cnt);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_BINDING_FAILED, "Failed binding freq_write_cnt "
+ LG_MSG_BINDING_FAILED, "Failed to bind freq_write_cnt "
"%d : %s", freq_write_cnt,
sqlite3_errmsg (sql_conn->sqlite3_db_conn));
ret = -1;
@@ -912,7 +962,7 @@ gf_sqlite3_find_recently_changed_files_freq (void *db_conn,
ret = sqlite3_bind_int64 (prep_stmt, 3, from_time_usec);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_BINDING_FAILED, "Failed binding from_time_usec "
+ LG_MSG_BINDING_FAILED, "Failed to bind from_time_usec "
"%"PRIu64" : %s", from_time_usec,
sqlite3_errmsg (sql_conn->sqlite3_db_conn));
ret = -1;
@@ -923,7 +973,7 @@ gf_sqlite3_find_recently_changed_files_freq (void *db_conn,
ret = sqlite3_bind_int (prep_stmt, 4, freq_read_cnt);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_BINDING_FAILED, "Failed binding freq_read_cnt "
+ LG_MSG_BINDING_FAILED, "Failed to bind freq_read_cnt "
"%d : %s", freq_read_cnt,
sqlite3_errmsg (sql_conn->sqlite3_db_conn));
ret = -1;
@@ -945,7 +995,7 @@ gf_sqlite3_find_recently_changed_files_freq (void *db_conn,
ret = gf_sql_clear_counters (sql_conn);
if (ret) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_CLEAR_COUNTER_FAILED, "Failed clearing"
+ LG_MSG_CLEAR_COUNTER_FAILED, "Failed to clear"
" counters!");
goto out;
}
@@ -953,6 +1003,8 @@ gf_sqlite3_find_recently_changed_files_freq (void *db_conn,
ret = 0;
out:
sqlite3_finalize (prep_stmt);
+ GF_FREE (base_query_str);
+ GF_FREE (query_str);
return ret;
}
@@ -983,15 +1035,17 @@ gf_sqlite3_find_unchanged_for_time_freq (void *db_conn,
gf_sql_connection_t *sql_conn = db_conn;
sqlite3_stmt *prep_stmt = NULL;
uint64_t for_time_usec = 0;
+ char *base_query_str = NULL;
CHECK_SQL_CONN (sql_conn, out);
- GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, query_callback, out);
+ GF_VALIDATE_OR_GOTO(GFDB_STR_SQLITE3, query_callback, out);
+
+ ret = gf_get_basic_query_stmt (&base_query_str);
+ if (ret <= 0) {
+ goto out;
+ }
- query_str = "select GF_FILE_TB.GF_ID,"
- " (select group_concat( GF_PID || ',' || FNAME || ','"
- " || FPATH || ',' || W_DEL_FLAG ||',' || LINK_UPDATE , '::')"
- " from GF_FLINK_TB where GF_FILE_TB.GF_ID = GF_FLINK_TB.GF_ID)"
- " from GF_FILE_TB where "
+ ret = gf_asprintf (&query_str, "%s AND "
/*First condition: For Writes
* Files that have write wind time smaller than for_time
* OR
@@ -1014,8 +1068,14 @@ gf_sqlite3_find_unchanged_for_time_freq (void *db_conn,
" OR "
"( (" GF_COL_TB_RFC " < ? ) AND"
"((" GF_COL_TB_RWSEC " * " TOSTRING(GFDB_MICROSEC) " + "
- GF_COL_TB_RWMSEC ") >= ? ) ) )";
+ GF_COL_TB_RWMSEC ") >= ? ) ) )", base_query_str);
+ if (ret < 0) {
+ gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_QUERY_FAILED,
+ "Failed to create query statement");
+ query_str = NULL;
+ goto out;
+ }
for_time_usec = gfdb_time_2_usec (for_time);
@@ -1023,7 +1083,7 @@ gf_sqlite3_find_unchanged_for_time_freq (void *db_conn,
&prep_stmt, 0);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_PREPARE_FAILED, "Failed preparing delete "
+ LG_MSG_PREPARE_FAILED, "Failed to prepare delete "
"statment %s : %s", query_str,
sqlite3_errmsg (sql_conn->sqlite3_db_conn));
ret = -1;
@@ -1034,7 +1094,7 @@ gf_sqlite3_find_unchanged_for_time_freq (void *db_conn,
ret = sqlite3_bind_int64 (prep_stmt, 1, for_time_usec);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_BINDING_FAILED, "Failed binding for_time_usec "
+ LG_MSG_BINDING_FAILED, "Failed to bind for_time_usec "
"%"PRIu64" : %s", for_time_usec,
sqlite3_errmsg (sql_conn->sqlite3_db_conn));
ret = -1;
@@ -1045,7 +1105,7 @@ gf_sqlite3_find_unchanged_for_time_freq (void *db_conn,
ret = sqlite3_bind_int (prep_stmt, 2, freq_write_cnt);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_BINDING_FAILED, "Failed binding freq_write_cnt"
+ LG_MSG_BINDING_FAILED, "Failed to bind freq_write_cnt"
" %d : %s", freq_write_cnt,
sqlite3_errmsg (sql_conn->sqlite3_db_conn));
ret = -1;
@@ -1056,7 +1116,7 @@ gf_sqlite3_find_unchanged_for_time_freq (void *db_conn,
ret = sqlite3_bind_int64 (prep_stmt, 3, for_time_usec);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_BINDING_FAILED, "Failed binding for_time_usec "
+ LG_MSG_BINDING_FAILED, "Failed to bind for_time_usec "
"%"PRIu64" : %s", for_time_usec,
sqlite3_errmsg (sql_conn->sqlite3_db_conn));
ret = -1;
@@ -1069,7 +1129,7 @@ gf_sqlite3_find_unchanged_for_time_freq (void *db_conn,
ret = sqlite3_bind_int64 (prep_stmt, 4, for_time_usec);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_BINDING_FAILED, "Failed binding for_time_usec "
+ LG_MSG_BINDING_FAILED, "Failed to bind for_time_usec "
"%"PRIu64" : %s", for_time_usec,
sqlite3_errmsg (sql_conn->sqlite3_db_conn));
ret = -1;
@@ -1080,7 +1140,7 @@ gf_sqlite3_find_unchanged_for_time_freq (void *db_conn,
ret = sqlite3_bind_int (prep_stmt, 5, freq_read_cnt);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_BINDING_FAILED, "Failed binding freq_read_cnt "
+ LG_MSG_BINDING_FAILED, "Failed to bind freq_read_cnt "
"%d : %s", freq_read_cnt,
sqlite3_errmsg (sql_conn->sqlite3_db_conn));
ret = -1;
@@ -1091,7 +1151,7 @@ gf_sqlite3_find_unchanged_for_time_freq (void *db_conn,
ret = sqlite3_bind_int64 (prep_stmt, 6, for_time_usec);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_BINDING_FAILED, "Failed binding for_time_usec "
+ LG_MSG_BINDING_FAILED, "Failed to bind for_time_usec "
"%"PRIu64" : %s", for_time_usec,
sqlite3_errmsg(sql_conn->sqlite3_db_conn));
ret = -1;
@@ -1112,7 +1172,7 @@ gf_sqlite3_find_unchanged_for_time_freq (void *db_conn,
ret = gf_sql_clear_counters (sql_conn);
if (ret) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_CLEAR_COUNTER_FAILED, "Failed clearing "
+ LG_MSG_CLEAR_COUNTER_FAILED, "Failed to clear "
"counters!");
goto out;
}
@@ -1121,6 +1181,8 @@ gf_sqlite3_find_unchanged_for_time_freq (void *db_conn,
ret = 0;
out:
sqlite3_finalize (prep_stmt);
+ GF_FREE (base_query_str);
+ GF_FREE (query_str);
return ret;
}
@@ -1136,8 +1198,8 @@ gf_sqlite3_clear_files_heat (void *db_conn)
ret = gf_sql_clear_counters (sql_conn);
if (ret) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_CLEAR_COUNTER_FAILED, "Failed clearing "
- "files heat!");
+ LG_MSG_CLEAR_COUNTER_FAILED, "Failed to clear "
+ "files heat");
goto out;
}
diff --git a/libglusterfs/src/gfdb/gfdb_sqlite3.h b/libglusterfs/src/gfdb/gfdb_sqlite3.h
index 5c259386a32..683f51be355 100644
--- a/libglusterfs/src/gfdb/gfdb_sqlite3.h
+++ b/libglusterfs/src/gfdb/gfdb_sqlite3.h
@@ -61,7 +61,6 @@ do {\
);;\
} while (0)
-
#define GF_COL_TB_WSEC GF_FILE_TABLE "." GF_COL_WSEC
#define GF_COL_TB_WMSEC GF_FILE_TABLE "." GF_COL_WMSEC
#define GF_COL_TB_UWSEC GF_FILE_TABLE "." GF_COL_UWSEC
diff --git a/libglusterfs/src/gfdb/gfdb_sqlite3_helper.c b/libglusterfs/src/gfdb/gfdb_sqlite3_helper.c
index 426fab0333e..e0a8b9c9803 100644
--- a/libglusterfs/src/gfdb/gfdb_sqlite3_helper.c
+++ b/libglusterfs/src/gfdb/gfdb_sqlite3_helper.c
@@ -1106,84 +1106,205 @@ gf_sql_query_function (sqlite3_stmt *prep_stmt,
gf_query_callback_t query_callback,
void *_query_cbk_args)
{
- int ret = -1;
- gfdb_query_record_t *gfdb_query_record = NULL;
- char *text_column = NULL;
- sqlite3 *db_conn = NULL;
+ int ret = -1;
+ gfdb_query_record_t *query_record = NULL;
+ char *text_column = NULL;
+ sqlite3 *db_conn = NULL;
+ uuid_t prev_gfid = {0};
+ uuid_t curr_gfid = {0};
+ uuid_t pgfid = {0};
+ char *base_name = NULL;
+ gf_boolean_t is_first_record = _gf_true;
+ gf_boolean_t is_query_empty = _gf_true;
GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, prep_stmt, out);
GF_VALIDATE_OR_GOTO (GFDB_STR_SQLITE3, query_callback, out);
db_conn = sqlite3_db_handle(prep_stmt);
- gfdb_query_record = gfdb_query_record_init ();
- if (!gfdb_query_record) {
- gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_CREATE_FAILED, "Failed to create "
- "gfdb_query_record");
- goto out;
- }
-
- /*Loop to access queried rows*/
+ /*
+ * Loop to access queried rows
+ * Each db record will have 3 columns
+ * GFID, PGFID, FILE_NAME
+ *
+ * For file with multiple hard links we will get multiple query rows
+ * with the same GFID, but different PGID and FILE_NAME Combination
+ * For Example if a file with
+ * GFID = 00000000-0000-0000-0000-000000000006
+ * has 3 hardlinks file1, file2 and file3 in 3 different folder
+ * with GFID's
+ * 00000000-0000-0000-0000-0000EFC00001,
+ * 00000000-0000-0000-0000-00000ABC0001 and
+ * 00000000-0000-0000-0000-00000ABC00CD
+ * Then there will be 3 records
+ * GFID : 00000000-0000-0000-0000-000000000006
+ * PGFID : 00000000-0000-0000-0000-0000EFC00001
+ * FILE_NAME : file1
+ *
+ * GFID : 00000000-0000-0000-0000-000000000006
+ * PGFID : 00000000-0000-0000-0000-00000ABC0001
+ * FILE_NAME : file2
+ *
+ * GFID : 00000000-0000-0000-0000-000000000006
+ * PGFID : 00000000-0000-0000-0000-00000ABC00CD
+ * FILE_NAME : file3
+ *
+ * This is retrieved and added to a single query_record
+ *
+ * query_record->gfid = 00000000-0000-0000-0000-000000000006
+ * ->link_info = {00000000-0000-0000-0000-0000EFC00001,
+ * "file1"}
+ * |
+ * V
+ * link_info = {00000000-0000-0000-0000-00000ABC0001,
+ * "file2"}
+ * |
+ * V
+ * link_info = {00000000-0000-0000-0000-00000ABC0001,
+ * "file3",
+ * list}
+ *
+ * This query record is sent to the registered query_callback()
+ *
+ * */
while ((ret = sqlite3_step (prep_stmt)) == SQLITE_ROW) {
- /*Clear the query record*/
- memset (gfdb_query_record, 0, sizeof(*gfdb_query_record));
-
if (sqlite3_column_count(prep_stmt) > 0) {
- /*Retriving GFID - column index is 0*/
+ is_query_empty = _gf_false;
+
+ /*Retrieving GFID - column index is 0*/
text_column = (char *)sqlite3_column_text
(prep_stmt, 0);
if (!text_column) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_GET_ID_FAILED, "Failed "
- "retriving GF_ID");
+ LG_MSG_GET_ID_FAILED, "Failed to"
+ "retrieve GFID");
+ goto out;
+ }
+ ret = gf_uuid_parse (text_column, curr_gfid);
+ if (ret) {
+ gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
+ LG_MSG_PARSE_FAILED, "Failed to parse "
+ "GFID");
+ goto out;
+ }
+
+ /*
+ * if the previous record was not of the current gfid
+ * call the call_back function and send the
+ * query record, which will have all the link_info
+ * objects associated with this gfid
+ *
+ * */
+ if (gf_uuid_compare (curr_gfid, prev_gfid) != 0) {
+
+ /* If this is not the first record */
+ if (!is_first_record) {
+ /*Call the call_back function provided*/
+ ret = query_callback (query_record,
+ _query_cbk_args);
+ if (ret) {
+ gf_msg (GFDB_STR_SQLITE3,
+ GF_LOG_ERROR, 0,
+ LG_MSG_QUERY_CALL_BACK_FAILED,
+ "Query call back "
+ "failed");
+ goto out;
+ }
+
+ }
+
+ /*Clear the query record*/
+ gfdb_query_record_free (query_record);
+ query_record = NULL;
+ query_record = gfdb_query_record_new ();
+ if (!query_record) {
+ gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR,
+ 0, LG_MSG_CREATE_FAILED,
+ "Failed to create "
+ "query_record");
+ goto out;
+ }
+
+ gf_uuid_copy(query_record->gfid,
+ curr_gfid);
+ gf_uuid_copy(prev_gfid, curr_gfid);
+
+ }
+
+ /* Get PGFID */
+ text_column = (char *)sqlite3_column_text
+ (prep_stmt, 1);
+ if (!text_column) {
+ gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
+ LG_MSG_GET_ID_FAILED, "Failed to"
+ " retrieve GF_ID");
goto out;
}
- ret = gf_uuid_parse (text_column, gfdb_query_record->gfid);
+ ret = gf_uuid_parse (text_column, pgfid);
if (ret) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_PARSE_FAILED, "Failed parsing "
+ LG_MSG_PARSE_FAILED, "Failed to parse "
"GF_ID");
goto out;
}
- /*Retrive Link Buffer - column index 1*/
+ /* Get Base name */
text_column = (char *)sqlite3_column_text
- (prep_stmt, 1);
- /* Get link string. Do shallow copy here
- * query_callback function should do a
- * deep copy and then do operations on this field*/
- gfdb_query_record->_link_info_str = text_column;
- gfdb_query_record->link_info_size = strlen
- (text_column);
-
- /* Call the call back function provided*/
- ret = query_callback (gfdb_query_record,
- _query_cbk_args);
+ (prep_stmt, 2);
+ if (!text_column) {
+ gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
+ LG_MSG_GET_ID_FAILED, "Failed to"
+ " retrieve GF_ID");
+ goto out;
+ }
+ base_name = text_column;
+
+
+ /* Add link info to the list */
+ ret = gfdb_add_link_to_query_record (query_record,
+ pgfid, base_name);
if (ret) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_QUERY_CALL_BACK_FAILED,
- "Query Call back failed!");
+ LG_MSG_GET_ID_FAILED, "Failed to"
+ " add link info to query record");
goto out;
}
+ is_first_record = _gf_false;
+
}
}
if (ret != SQLITE_DONE) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
- LG_MSG_GET_RECORD_FAILED, "Failed retriving records "
+ LG_MSG_GET_RECORD_FAILED, "Failed to retrieve records "
"from db : %s", sqlite3_errmsg (db_conn));
ret = -1;
goto out;
}
+
+ if (!is_query_empty) {
+ /*
+ * Call the call_back function for the last record from the
+ * Database
+ * */
+ ret = query_callback (query_record, _query_cbk_args);
+ if (ret) {
+ gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0,
+ LG_MSG_QUERY_CALL_BACK_FAILED,
+ "Query call back failed");
+ goto out;
+ }
+ }
+
ret = 0;
out:
- gfdb_query_record_fini (&gfdb_query_record);
+ gfdb_query_record_free (query_record);
+ query_record = NULL;
return ret;
}
@@ -1207,7 +1328,7 @@ gf_sql_clear_counters (gf_sql_connection_t *sql_conn)
&sql_strerror);
if (ret != SQLITE_OK) {
gf_msg (GFDB_STR_SQLITE3, GF_LOG_ERROR, 0, LG_MSG_EXEC_FAILED,
- "Failed executing: %s : %s",
+ "Failed to execute: %s : %s",
query_str, sql_strerror);
sqlite3_free (sql_strerror);
ret = -1;
diff --git a/xlators/cluster/dht/src/tier.c b/xlators/cluster/dht/src/tier.c
index 618b82d18ee..4af5e67ebf2 100644
--- a/xlators/cluster/dht/src/tier.c
+++ b/xlators/cluster/dht/src/tier.c
@@ -34,49 +34,6 @@ static gfdb_methods_t gfdb_methods;
static int
-tier_parse_query_str (char *query_record_str,
- char *gfid, char *link_buffer, ssize_t *link_size)
-{
- char *token_str = NULL;
- char *delimiter = "|";
- char *saveptr = NULL;
- int ret = -1;
-
- GF_VALIDATE_OR_GOTO ("tier", query_record_str, out);
- GF_VALIDATE_OR_GOTO ("tier", gfid, out);
- GF_VALIDATE_OR_GOTO ("tier", link_buffer, out);
- GF_VALIDATE_OR_GOTO ("tier", link_size, out);
-
- token_str = strtok_r (query_record_str, delimiter, &saveptr);
- if (!token_str)
- goto out;
-
- strcpy (gfid, token_str);
-
-
- token_str = strtok_r (NULL, delimiter, &saveptr);
- if (!token_str)
- goto out;
-
- strcpy (link_buffer, token_str);
-
- token_str = strtok_r (NULL, delimiter, &saveptr);
- if (!token_str)
- goto out;
-
- *link_size = atoi (token_str);
-
- ret = 0;
-out:
- return ret;
-}
-
-/*
- * return 0 if the same node.
- * return 1 if not the same node, but no errors.
- * return -1 if errors.xs
- */
-static int
tier_check_same_node (xlator_t *this, loc_t *loc, gf_defrag_info_t *defrag)
{
int ret = -1;
@@ -241,14 +198,9 @@ static int
tier_migrate_using_query_file (void *_args)
{
int ret = -1;
- char gfid_str[UUID_CANONICAL_FORM_LEN+1] = "";
- char query_record_str[4096] = "";
query_cbk_args_t *query_cbk_args = (query_cbk_args_t *) _args;
xlator_t *this = NULL;
gf_defrag_info_t *defrag = NULL;
- char *token_str = NULL;
- char *delimiter = "::";
- char *link_buffer = NULL;
gfdb_query_record_t *query_record = NULL;
gfdb_link_info_t *link_info = NULL;
struct iatt par_stbuf = {0,};
@@ -256,6 +208,9 @@ tier_migrate_using_query_file (void *_args)
loc_t p_loc = {0,};
loc_t loc = {0,};
dict_t *migrate_data = NULL;
+ dict_t *xdata_request = NULL;
+ dict_t *xdata_response = NULL;
+ char *parent_path = NULL;
inode_t *linked_inode = NULL;
/*
* per_file_status and per_link_status
@@ -266,8 +221,7 @@ tier_migrate_using_query_file (void *_args)
int per_file_status = 0;
int per_link_status = 0;
int total_status = 0;
- FILE *queryFILE = NULL;
- char *link_str = NULL;
+ int query_fd = 0;
xlator_t *src_subvol = NULL;
dht_conf_t *conf = NULL;
uint64_t total_migrated_bytes = 0;
@@ -277,75 +231,72 @@ tier_migrate_using_query_file (void *_args)
GF_VALIDATE_OR_GOTO ("tier", query_cbk_args->this, out);
this = query_cbk_args->this;
GF_VALIDATE_OR_GOTO (this->name, query_cbk_args->defrag, out);
- GF_VALIDATE_OR_GOTO (this->name, query_cbk_args->queryFILE, out);
+ GF_VALIDATE_OR_GOTO (this->name, (query_cbk_args->query_fd > 0), out);
GF_VALIDATE_OR_GOTO (this->name, this->private, out);
conf = this->private;
defrag = query_cbk_args->defrag;
- queryFILE = query_cbk_args->queryFILE;
+ query_fd = query_cbk_args->query_fd;
+
+ migrate_data = dict_new ();
+ if (!migrate_data)
+ goto out;
- query_record = gfdb_query_record_init ();
- if (!query_record) {
+ xdata_request = dict_new ();
+ if (!xdata_request) {
gf_msg (this->name, GF_LOG_ERROR, 0,
DHT_MSG_LOG_TIER_ERROR,
- "Call to gfdb_query_record_init() failed.");
+ "Failed to create xdata_request dict");
goto out;
}
-
- query_record->_link_info_str = GF_CALLOC (1, DB_QUERY_RECORD_SIZE,
- gf_common_mt_char);
- if (!query_record->_link_info_str) {
+ ret = dict_set_int32 (xdata_request,
+ GET_ANCESTRY_PATH_KEY, 42);
+ if (ret) {
gf_msg (this->name, GF_LOG_ERROR, 0,
DHT_MSG_LOG_TIER_ERROR,
- "Allocating query record link info string failed.");
+ "Failed to set value to dict : key %s \n",
+ GET_ANCESTRY_PATH_KEY);
goto out;
}
- link_buffer = query_record->_link_info_str;
- link_info = gfdb_link_info_init ();
-
- migrate_data = dict_new ();
- if (!migrate_data)
- goto out;
/* Per file */
- while (fgets (query_record_str, sizeof (query_record_str), queryFILE)) {
+ while ((ret = gfdb_methods.gfdb_read_query_record
+ (query_fd, &query_record)) != 0) {
+
+ if (ret < 0) {
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ DHT_MSG_LOG_TIER_ERROR,
+ "Failed to fetch query record "
+ "from query file");
+ goto out;
+ }
per_file_status = 0;
per_link_status = 0;
+ dict_del (migrate_data, GF_XATTR_FILE_MIGRATE_KEY);
+
+ dict_del (migrate_data, "from.migrator");
+
if (defrag->tier_conf.request_pause) {
gf_msg (this->name, GF_LOG_INFO, 0,
DHT_MSG_LOG_TIER_STATUS,
- "Tiering paused. Exiting tier_migrate_using_query_file");
+ "Tiering paused. "
+ "Exiting tier_migrate_using_query_file");
break;
}
- memset (gfid_str, 0, sizeof (gfid_str));
- memset (query_record->_link_info_str, 0, DB_QUERY_RECORD_SIZE);
-
- if (tier_parse_query_str (query_record_str, gfid_str,
- link_buffer,
- &query_record->link_info_size)) {
- gf_msg (this->name, GF_LOG_ERROR, 0,
- DHT_MSG_LOG_TIER_ERROR,
- "failed parsing %s\n", query_record_str);
+ if (!tier_do_migration (this, query_cbk_args->is_promotion)) {
+ gfdb_methods.gfdb_query_record_free (query_record);
+ query_record = NULL;
continue;
}
- if (!tier_do_migration (this, query_cbk_args->is_promotion))
- continue;
-
- gf_uuid_parse (gfid_str, query_record->gfid);
-
- dict_del (migrate_data, GF_XATTR_FILE_MIGRATE_KEY);
-
- dict_del (migrate_data, "from.migrator");
- token_str = strtok (link_buffer, delimiter);
- if (token_str != NULL) {
+ if (!list_empty (&query_record->link_list)) {
per_file_status =
dict_set_str (migrate_data,
GF_XATTR_FILE_MIGRATE_KEY,
@@ -381,48 +332,29 @@ tier_migrate_using_query_file (void *_args)
}
per_link_status = 0;
- /* Per link of file */
- while (token_str != NULL) {
-
- if (defrag->tier_conf.request_pause) {
- gf_msg (this->name, GF_LOG_INFO, 0,
- DHT_MSG_LOG_TIER_STATUS,
- "Tiering paused. "
- "Exiting tier_migrate_using_query_file");
- goto abort;
- }
-
- link_str = gf_strdup (token_str);
-
- if (!link_info) {
- per_link_status = -1;
- goto per_file_out;
- }
-
- memset (link_info, 0, sizeof(gfdb_link_info_t));
-
- ret = str_to_link_info (link_str, link_info);
- if (ret) {
- gf_msg (this->name, GF_LOG_ERROR, 0,
- DHT_MSG_LOG_TIER_ERROR,
- "failed parsing %s\n", link_str);
- per_link_status = -1;
- goto abort;
- }
+ /* For now we only support single link migration. And we will
+ * ignore other hard links in the link info list of query record
+ * TODO: Multiple hard links migration */
+ if (!list_empty (&query_record->link_list)) {
+ link_info = list_first_entry
+ (&query_record->link_list,
+ gfdb_link_info_t, list);
+ }
+ if (link_info != NULL) {
+ /* Lookup for parent and get the path of parent */
gf_uuid_copy (p_loc.gfid, link_info->pargfid);
-
p_loc.inode = inode_new (defrag->root_inode->table);
if (!p_loc.inode) {
gf_msg (this->name, GF_LOG_ERROR, 0,
DHT_MSG_LOG_TIER_ERROR,
- "failed parsing %s\n", link_str);
+ "Failed to create reference to inode");
per_link_status = -1;
goto abort;
}
ret = syncop_lookup (this, &p_loc, &par_stbuf, NULL,
- NULL, NULL);
+ xdata_request, &xdata_response);
if (ret) {
gf_msg (this->name, GF_LOG_ERROR, -ret,
DHT_MSG_LOG_TIER_ERROR,
@@ -430,42 +362,60 @@ tier_migrate_using_query_file (void *_args)
per_link_status = -1;
goto abort;
}
+ ret = dict_get_str (xdata_response,
+ GET_ANCESTRY_PATH_KEY,
+ &parent_path);
+ if (ret || !parent_path) {
+ gf_msg (this->name, GF_LOG_ERROR, 0,
+ DHT_MSG_LOG_TIER_ERROR,
+ "Failed to get parent path\n");
+ per_link_status = -1;
+ goto abort;
+ }
+
linked_inode = inode_link (p_loc.inode, NULL, NULL,
&par_stbuf);
inode_unref (p_loc.inode);
p_loc.inode = linked_inode;
+
+
+
+ /* Preparing File Inode */
gf_uuid_copy (loc.gfid, query_record->gfid);
loc.inode = inode_new (defrag->root_inode->table);
gf_uuid_copy (loc.pargfid, link_info->pargfid);
loc.parent = inode_ref (p_loc.inode);
+ /* Get filename and Construct file path */
loc.name = gf_strdup (link_info->file_name);
if (!loc.name) {
gf_msg (this->name, GF_LOG_ERROR, 0,
- DHT_MSG_LOG_TIER_ERROR, "ERROR in "
- "memory allocation\n");
+ DHT_MSG_LOG_TIER_ERROR, "Memory "
+ "allocation failed.\n");
per_link_status = -1;
goto abort;
}
-
- loc.path = gf_strdup (link_info->file_path);
- if (!loc.path) {
+ ret = gf_asprintf((char **)&(loc.path), "%s/%s",
+ parent_path, loc.name);
+ if (ret < 0) {
gf_msg (this->name, GF_LOG_ERROR, 0,
- DHT_MSG_LOG_TIER_ERROR, "ERROR in "
- "memory allocation\n");
+ DHT_MSG_LOG_TIER_ERROR, "Failed to "
+ "construct file path for %s %s\n",
+ parent_path, loc.name);
per_link_status = -1;
goto abort;
}
gf_uuid_copy (loc.parent->gfid, link_info->pargfid);
+ /* lookup file inode */
ret = syncop_lookup (this, &loc, &current, NULL,
NULL, NULL);
if (ret) {
- gf_msg (this->name, GF_LOG_ERROR, 0,
- DHT_MSG_LOG_TIER_ERROR, "ERROR in "
- "current lookup\n");
+ gf_msg (this->name, GF_LOG_ERROR, -ret,
+ DHT_MSG_LOG_TIER_ERROR, "Failed to do "
+ "lookup on file %s\n", loc.name);
per_link_status = -1;
goto abort;
}
@@ -474,6 +424,7 @@ tier_migrate_using_query_file (void *_args)
inode_unref (loc.inode);
loc.inode = linked_inode;
+
/*
* Do not promote/demote if file already is where it
* should be. It means another brick moved the file
@@ -509,7 +460,7 @@ tier_migrate_using_query_file (void *_args)
goto abort;
}
ret = 0;
- /* By setting per_linl_status to 1 we are
+ /* By setting per_link_status to 1 we are
* ignoring this status and will not be counting
* this file for migration */
per_link_status = 1;
@@ -522,18 +473,18 @@ tier_migrate_using_query_file (void *_args)
gf_msg (this->name, GF_LOG_INFO, 0,
DHT_MSG_LOG_TIER_STATUS,
"Tiering paused. "
- "Exiting tier_migrate_using_query_file");
+ "Exiting "
+ "tier_migrate_using_query_file");
goto abort;
}
+ /* Data migration */
ret = syncop_setxattr (this, &loc, migrate_data, 0,
NULL, NULL);
if (ret) {
- gf_msg (this->name, GF_LOG_ERROR, 0,
- DHT_MSG_LOG_TIER_ERROR, "ERROR %d in "
- "current migration %s %s\n", ret,
- loc.name,
- loc.path);
+ gf_msg (this->name, GF_LOG_ERROR, -ret,
+ DHT_MSG_LOG_TIER_ERROR, "Failed to "
+ "migrate %s \n", loc.name);
per_link_status = -1;
goto abort;
}
@@ -562,6 +513,11 @@ tier_migrate_using_query_file (void *_args)
defrag->tier_conf.blocks_total;
pthread_mutex_unlock (&dm_stat_mutex);
}
+abort:
+ GF_FREE ((char *) loc.name);
+ loc.name = NULL;
+ loc_wipe (&loc);
+ loc_wipe (&p_loc);
if ((++total_files > defrag->tier_conf.max_migrate_files) ||
(total_migrated_bytes > defrag->tier_conf.max_migrate_bytes)) {
@@ -573,15 +529,6 @@ tier_migrate_using_query_file (void *_args)
total_files);
goto out;
}
-
-abort:
- loc_wipe(&loc);
- loc_wipe(&p_loc);
-
- token_str = NULL;
- token_str = strtok (NULL, delimiter);
- GF_FREE (link_str);
-
}
per_file_status = per_link_status;
per_file_out:
@@ -599,36 +546,45 @@ per_file_out:
total_status = total_status + per_file_status;
per_link_status = 0;
per_file_status = 0;
- query_record_str[0] = '\0';
+
+ gfdb_methods.gfdb_query_record_free (query_record);
+ query_record = NULL;
}
out:
- if (link_buffer)
- GF_FREE (link_buffer);
- gfdb_link_info_fini (&link_info);
+ if (xdata_request) {
+ dict_unref (xdata_request);
+ }
+
if (migrate_data)
dict_unref (migrate_data);
- gfdb_query_record_fini (&query_record);
+
+
+ gfdb_methods.gfdb_query_record_free (query_record);
+ query_record = NULL;
+
return total_status;
}
-/*This is the call back function per record/file from data base*/
+/* This is the call back function per record/file from data base */
static int
tier_gf_query_callback (gfdb_query_record_t *gfdb_query_record,
void *_args) {
int ret = -1;
- char gfid_str[UUID_CANONICAL_FORM_LEN+1] = "";
query_cbk_args_t *query_cbk_args = _args;
GF_VALIDATE_OR_GOTO ("tier", query_cbk_args, out);
GF_VALIDATE_OR_GOTO ("tier", query_cbk_args->defrag, out);
- GF_VALIDATE_OR_GOTO ("tier", query_cbk_args->queryFILE, out);
+ GF_VALIDATE_OR_GOTO ("tier", (query_cbk_args->query_fd > 0), out);
- gf_uuid_unparse (gfdb_query_record->gfid, gfid_str);
- fprintf (query_cbk_args->queryFILE, "%s|%s|%zd\n", gfid_str,
- gfdb_query_record->_link_info_str,
- gfdb_query_record->link_info_size);
+ ret = gfdb_methods.gfdb_write_query_record (query_cbk_args->query_fd,
+ gfdb_query_record);
+ if (ret) {
+ gf_msg ("tier", GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
+ "Failed writing query record to query file");
+ goto out;
+ }
pthread_mutex_lock (&dm_stat_mutex);
query_cbk_args->defrag->num_files_lookedup++;
@@ -642,7 +598,7 @@ out:
-/*Create query file in tier process*/
+/* Create query file in tier process */
static int
tier_process_self_query (tier_brick_list_t *local_brick, void *args)
{
@@ -653,17 +609,17 @@ tier_process_self_query (tier_brick_list_t *local_brick, void *args)
gfdb_conn_node_t *conn_node = NULL;
dict_t *params_dict = NULL;
dict_t *ctr_ipc_dict = NULL;
- _gfdb_brick_dict_info_t *gfdb_brick_dict_info = args;
+ gfdb_brick_info_t *gfdb_brick_info = args;
/*Init of all the essentials*/
- GF_VALIDATE_OR_GOTO ("tier", gfdb_brick_dict_info , out);
- query_cbk_args = gfdb_brick_dict_info->_query_cbk_args;
+ GF_VALIDATE_OR_GOTO ("tier", gfdb_brick_info , out);
+ query_cbk_args = gfdb_brick_info->_query_cbk_args;
GF_VALIDATE_OR_GOTO ("tier", query_cbk_args->this, out);
this = query_cbk_args->this;
GF_VALIDATE_OR_GOTO (this->name,
- gfdb_brick_dict_info->_query_cbk_args, out);
+ gfdb_brick_info->_query_cbk_args, out);
GF_VALIDATE_OR_GOTO (this->name, local_brick, out);
@@ -693,31 +649,33 @@ tier_process_self_query (tier_brick_list_t *local_brick, void *args)
goto out;
}
- /*Query for eligible files from db*/
- query_cbk_args->queryFILE = fopen (
- GET_QFILE_PATH (gfdb_brick_dict_info->_gfdb_promote), "a+");
- if (!query_cbk_args->queryFILE) {
+ /* Query for eligible files from db */
+ query_cbk_args->query_fd = open (GET_QFILE_PATH
+ (gfdb_brick_info->_gfdb_promote),
+ O_WRONLY | O_CREAT | O_APPEND,
+ S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+ if (query_cbk_args->query_fd < 0) {
gf_msg (this->name, GF_LOG_ERROR, errno,
DHT_MSG_LOG_TIER_ERROR,
"Failed to open query file %s",
GET_QFILE_PATH
- (gfdb_brick_dict_info->_gfdb_promote));
+ (gfdb_brick_info->_gfdb_promote));
goto out;
}
- if (!gfdb_brick_dict_info->_gfdb_promote) {
+ if (!gfdb_brick_info->_gfdb_promote) {
if (query_cbk_args->defrag->write_freq_threshold == 0 &&
query_cbk_args->defrag->read_freq_threshold == 0) {
ret = gfdb_methods.find_unchanged_for_time (
conn_node,
tier_gf_query_callback,
(void *)query_cbk_args,
- gfdb_brick_dict_info->time_stamp);
+ gfdb_brick_info->time_stamp);
} else {
ret = gfdb_methods.find_unchanged_for_time_freq (
conn_node,
tier_gf_query_callback,
(void *)query_cbk_args,
- gfdb_brick_dict_info->time_stamp,
+ gfdb_brick_info->time_stamp,
query_cbk_args->defrag->
write_freq_threshold,
query_cbk_args->defrag->
@@ -731,13 +689,13 @@ tier_process_self_query (tier_brick_list_t *local_brick, void *args)
conn_node,
tier_gf_query_callback,
(void *)query_cbk_args,
- gfdb_brick_dict_info->time_stamp);
+ gfdb_brick_info->time_stamp);
} else {
ret = gfdb_methods.find_recently_changed_files_freq (
conn_node,
tier_gf_query_callback,
(void *)query_cbk_args,
- gfdb_brick_dict_info->time_stamp,
+ gfdb_brick_info->time_stamp,
query_cbk_args->defrag->
write_freq_threshold,
query_cbk_args->defrag->read_freq_threshold,
@@ -786,9 +744,9 @@ out:
ctr_ipc_dict = NULL;
}
- if (query_cbk_args && query_cbk_args->queryFILE) {
- fclose (query_cbk_args->queryFILE);
- query_cbk_args->queryFILE = NULL;
+ if (query_cbk_args && query_cbk_args->query_fd >= 0) {
+ close (query_cbk_args->query_fd);
+ query_cbk_args->query_fd = -1;
}
gfdb_methods.fini_db (conn_node);
return ret;
@@ -805,19 +763,19 @@ tier_process_ctr_query (tier_brick_list_t *local_brick, void *args)
xlator_t *this = NULL;
dict_t *ctr_ipc_in_dict = NULL;
dict_t *ctr_ipc_out_dict = NULL;
- _gfdb_brick_dict_info_t *gfdb_brick_dict_info = args;
+ gfdb_brick_info_t *gfdb_brick_info = args;
gfdb_ipc_ctr_params_t *ipc_ctr_params = NULL;
int count = 0;
/*Init of all the essentials*/
- GF_VALIDATE_OR_GOTO ("tier", gfdb_brick_dict_info , out);
- query_cbk_args = gfdb_brick_dict_info->_query_cbk_args;
+ GF_VALIDATE_OR_GOTO ("tier", gfdb_brick_info , out);
+ query_cbk_args = gfdb_brick_info->_query_cbk_args;
GF_VALIDATE_OR_GOTO ("tier", query_cbk_args->this, out);
this = query_cbk_args->this;
GF_VALIDATE_OR_GOTO (this->name,
- gfdb_brick_dict_info->_query_cbk_args, out);
+ gfdb_brick_info->_query_cbk_args, out);
GF_VALIDATE_OR_GOTO (this->name, local_brick, out);
@@ -842,13 +800,13 @@ tier_process_ctr_query (tier_brick_list_t *local_brick, void *args)
}
/* set all the query params*/
- ipc_ctr_params->is_promote = gfdb_brick_dict_info->_gfdb_promote;
+ ipc_ctr_params->is_promote = gfdb_brick_info->_gfdb_promote;
ipc_ctr_params->write_freq_threshold = query_cbk_args->
defrag->write_freq_threshold;
ipc_ctr_params->read_freq_threshold = query_cbk_args->
defrag->read_freq_threshold;
memcpy (&ipc_ctr_params->time_stamp,
- gfdb_brick_dict_info->time_stamp,
+ gfdb_brick_info->time_stamp,
sizeof (gfdb_time_t));
SET_DB_PARAM_TO_DICT(this->name, ctr_ipc_in_dict,
@@ -1037,7 +995,7 @@ tier_build_migration_qfile (demotion_args_t *args,
gf_boolean_t is_promotion)
{
gfdb_time_t current_time;
- _gfdb_brick_dict_info_t gfdb_brick_dict_info;
+ gfdb_brick_info_t gfdb_brick_info;
gfdb_time_t time_in_past;
int ret = -1;
tier_brick_list_t *local_brick = NULL;
@@ -1074,13 +1032,13 @@ tier_build_migration_qfile (demotion_args_t *args,
/* away this problem by always setting usec to 0. */
time_in_past.tv_usec = 0;
- gfdb_brick_dict_info.time_stamp = &time_in_past;
- gfdb_brick_dict_info._gfdb_promote = is_promotion;
- gfdb_brick_dict_info._query_cbk_args = query_cbk_args;
+ gfdb_brick_info.time_stamp = &time_in_past;
+ gfdb_brick_info._gfdb_promote = is_promotion;
+ gfdb_brick_info._query_cbk_args = query_cbk_args;
list_for_each_entry (local_brick, args->brick_list, list) {
ret = tier_process_brick (local_brick,
- &gfdb_brick_dict_info);
+ &gfdb_brick_info);
if (ret) {
gf_msg (args->this->name, GF_LOG_ERROR, 0,
DHT_MSG_BRICK_QUERY_FAILED,
@@ -1101,16 +1059,16 @@ tier_migrate_files_using_qfile (demotion_args_t *comp,
char renamed_file[PATH_MAX] = "";
int ret = -1;
- query_cbk_args->queryFILE = fopen (qfile, "r");
- if (!query_cbk_args->queryFILE) {
- gf_msg ("tier", GF_LOG_ERROR, 0,
+ query_cbk_args->query_fd = open (qfile, O_RDONLY);
+ if (query_cbk_args->query_fd < 0) {
+ gf_msg ("tier", GF_LOG_ERROR, errno,
DHT_MSG_FOPEN_FAILED,
- "Failed opening %s for migration", qfile);
+ "Failed to open %s for migration", qfile);
goto out;
}
ret = tier_migrate_using_query_file ((void *)query_cbk_args);
- fclose (query_cbk_args->queryFILE);
- query_cbk_args->queryFILE = NULL;
+ close (query_cbk_args->query_fd);
+ query_cbk_args->query_fd = -1;
if (ret) {
sprintf (renamed_file, "%s.err", qfile);
rename (qfile, renamed_file);
diff --git a/xlators/cluster/dht/src/tier.h b/xlators/cluster/dht/src/tier.h
index b840f339d2e..8087eee87b3 100644
--- a/xlators/cluster/dht/src/tier.h
+++ b/xlators/cluster/dht/src/tier.h
@@ -44,18 +44,18 @@
typedef struct _query_cbk_args {
xlator_t *this;
gf_defrag_info_t *defrag;
- FILE *queryFILE;
+ int query_fd;
int is_promotion;
} query_cbk_args_t;
int
gf_run_tier(xlator_t *this, gf_defrag_info_t *defrag);
-typedef struct _gfdb_brick_dict_info {
+typedef struct gfdb_brick_info {
gfdb_time_t *time_stamp;
gf_boolean_t _gfdb_promote;
query_cbk_args_t *_query_cbk_args;
-} _gfdb_brick_dict_info_t;
+} gfdb_brick_info_t;
typedef struct brick_list {
xlator_t *xlator;
diff --git a/xlators/features/changetimerecorder/src/changetimerecorder.c b/xlators/features/changetimerecorder/src/changetimerecorder.c
index fb337674937..c4634c16ee3 100644
--- a/xlators/features/changetimerecorder/src/changetimerecorder.c
+++ b/xlators/features/changetimerecorder/src/changetimerecorder.c
@@ -1386,15 +1386,18 @@ static int
ctr_db_query_callback (gfdb_query_record_t *gfdb_query_record,
void *args) {
int ret = -1;
- char gfid_str[UUID_CANONICAL_FORM_LEN+1] = "";
ctr_query_cbk_args_t *query_cbk_args = args;
GF_VALIDATE_OR_GOTO ("ctr", query_cbk_args, out);
- gf_uuid_unparse (gfdb_query_record->gfid, gfid_str);
- fprintf (query_cbk_args->queryFILE, "%s|%s|%ld\n", gfid_str,
- gfdb_query_record->_link_info_str,
- gfdb_query_record->link_info_size);
+ ret = gfdb_write_query_record (query_cbk_args->query_fd,
+ gfdb_query_record);
+ if (ret) {
+ gf_msg ("ctr", GF_LOG_ERROR, 0,
+ CTR_MSG_FATAL_ERROR,
+ "Failed to write to query file");
+ goto out;
+ }
query_cbk_args->count++;
@@ -1429,8 +1432,10 @@ ctr_db_query (xlator_t *this,
GF_VALIDATE_OR_GOTO (this->name, ipc_ctr_params, out);
/*Query for eligible files from db*/
- query_cbk_args.queryFILE = fopen(query_file, "a+");
- if (!query_cbk_args.queryFILE) {
+ query_cbk_args.query_fd = open (query_file,
+ O_WRONLY | O_CREAT | O_APPEND,
+ S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+ if (query_cbk_args.query_fd < 0) {
gf_msg (this->name, GF_LOG_ERROR, errno,
CTR_MSG_FATAL_ERROR,
"Failed to open query file %s", query_file);
@@ -1494,9 +1499,9 @@ out:
if (!ret)
ret = query_cbk_args.count;
- if (query_cbk_args.queryFILE) {
- fclose (query_cbk_args.queryFILE);
- query_cbk_args.queryFILE = NULL;
+ if (query_cbk_args.query_fd >= 0) {
+ close (query_cbk_args.query_fd);
+ query_cbk_args.query_fd = -1;
}
return ret;
diff --git a/xlators/features/changetimerecorder/src/ctr-helper.h b/xlators/features/changetimerecorder/src/ctr-helper.h
index 654607fb852..e833c872881 100644
--- a/xlators/features/changetimerecorder/src/ctr-helper.h
+++ b/xlators/features/changetimerecorder/src/ctr-helper.h
@@ -37,7 +37,7 @@
typedef struct ctr_query_cbk_args {
- FILE *queryFILE;
+ int query_fd;
int count;
} ctr_query_cbk_args_t;