cluster/tier: Adding compaction option for metadata databases

Problem: As metadata in the database fills up, querying the database
take a long time. As a result, tier migration slows down.  To
counteract this, we added a way to enable the compaction methods of
the underlying database. The goal is to reduce the size of the
underlying file by eliminating database fragmentation.

NOTE: There is currently a bug where sometimes a brick will
attempt to activate compaction. This happens even compaction is already
turned on.

The cause is narrowed down to the compact_mode_switch flipping its value.

Changes: libglusterfs/src/gfdb - Added a gfdb function to compact the
underlying database, compact_db() This is a no-op if the database has
no such option.

- Added a compaction function for SQLite3 that does the following

1) Changes the auto_vacuum pragma of the database
2) Compacts the database according to the type of compaction requested

- Compaction type can be changed by changing the macro
  GF_SQL_COMPACT_DEF to one of the 4 compaction types in
  gfdb_sqlite3.h

  It is currently set to GF_SQL_COMPACT_INCR, or incremental
  vacuuming.

xlators/cluster/dht/src - Added the following command-line option to
enable SQLite3 compaction.

gluster volume set <vol-name> tier-compact <off|on>

- Added the following command-line option to change the frequency the
  hot and cold tier are ordered to compact.

gluster volume set <vol-name> tier-hot-compact-frequency <int>
gluster volume set <vol-name> tier-cold-compact-frequency <int>

- tier daemon periodically sends the (new)
  GFDB_IPC_CTR_SET_COMPACT_PRAGMA IPC to the CTR xlator. The IPC
  triggers compaction of the database.

  The inputs are both gf_boolean_t.

  IPC Input:

  compact_active: Is compaction currently on for the db.
  compact_mode_switched: Did we flip the compaction switch recently?

  IPC Output:

  0 if the compaction succeeds.
  Non-zero otherwise.

xlators/features/changetimerecorder/src/ - When the CTR gets the
compaction IPC, it launches a thread that will perform the
compaction. The IPC ends after the thread is launched. To avoid extra
allocations, the parameters are passed using static variables.

Change-Id: I5e1433becb9eeff2afe8dcb4a5798977bf5ba0dd
Signed-off-by: Diogenes Nunez <dnunez@redhat.com>
Reviewed-on: http://review.gluster.org/15031
Reviewed-by: Milind Changire <mchangir@redhat.com>
Reviewed-by: Dan Lambright <dlambrig@redhat.com>
Tested-by: Dan Lambright <dlambrig@redhat.com>
CentOS-regression: Gluster Build System <jenkins@build.gluster.org>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
Smoke: Gluster Build System <jenkins@build.gluster.org>

1 files changed, 17 insertions, 33 deletions

diff --git a/libglusterfs/src/gfdb/gfdb_data_store_types.h b/libglusterfs/src/gfdb/gfdb_data_store_types.h
index 1acbdf2f99f..d0c96370eb8 100644
--- a/libglusterfs/src/gfdb/gfdb_data_store_types.h
+++ b/libglusterfs/src/gfdb/gfdb_data_store_types.h
@@ -40,7 +40,8 @@ typedef enum gf_db_operation {
         GFDB_W_DELETE_DB_OP,
         GFDB_UW_DELETE_DB_OP,
         GFDB_WFC_UPDATE_DB_OP,
-        GFDB_RFC_UPDATE_DB_OP
+        GFDB_RFC_UPDATE_DB_OP,
+        GFDB_DB_COMPACT_DB_OP /* Added for VACUUM/manual compaction support */
 } gf_db_operation_t;
 
 
@@ -81,19 +82,12 @@ gfdb_time_2_usec(gfdb_time_t *gfdb_time)
         return ((uint64_t) gfdb_time->tv_sec * GFDB_MICROSEC) + gfdb_time->tv_usec;
 }
 
-
-
-
-
 /******************************************************************************
  *
  *              Insert/Update Record related data structures/functions
  *
  * ****************************************************************************/
 
-
-
-
 /*Indicated a generic synchronous write to the db
  * This may or may not be implemented*/
 typedef enum gfdb_sync_type {
@@ -123,11 +117,6 @@ out:
         return ret;
 }
 
-
-
-
-
-
 /*Indicated different types of db*/
 typedef enum gfdb_db_type {
         GFDB_INVALID_DB = -1,
@@ -165,12 +154,6 @@ out:
         return ret;
 }
 
-
-
-
-
-
-
 /*Tells the path of the fop*/
 typedef enum gfdb_fop_path {
         GFDB_FOP_INVALID = -1,
@@ -206,12 +189,6 @@ isunwindpath(gfdb_fop_path_t gfdb_fop_path)
         return (gfdb_fop_path >= GFDB_FOP_UNWIND) ? _gf_true : _gf_false;
 }
 
-
-
-
-
-
-
 /*Tell what type of fop it was
  * Like whether a dentry fop or a inode fop
  * Read fop or a write fop etc*/
@@ -258,12 +235,6 @@ isdentrycreatefop(gfdb_fop_type_t fop_type)
                         _gf_true : _gf_false;
 }
 
-
-
-
-
-
-
 /*The structure that is used to send insert/update the databases
  * using insert_db api*/
 typedef struct gfdb_db_record {
@@ -374,6 +345,20 @@ typedef int
 
 
 
+/*Used to compact the database
+ * Arguments:
+ *      db_conn                        :  GFDB Connection node
+ *      compact_active                 :  Is compaction currently on?
+ *      compact_mode_switched          :  Was the compaction switch flipped?
+ * Returns : if successful return 0 or
+ *          -ve value in case of failure*/
+typedef int
+(*gfdb_compact_db_t)(void *db_conn, gf_boolean_t compact_active,
+                     gf_boolean_t compact_mode_switched);
+
+
+
+
 /* Query all the records from the database
  * Arguments:
  *      db_conn        : plugin specific data base connection
@@ -502,6 +487,7 @@ typedef struct gfdb_db_operations {
         gfdb_fini_db_t                        fini_db_op;
         gfdb_insert_record_t                  insert_record_op;
         gfdb_delete_record_t                  delete_record_op;
+        gfdb_compact_db_t                     compact_db_op;
         gfdb_find_all_t                       find_all_op;
         gfdb_find_unchanged_for_time_t        find_unchanged_for_time_op;
         gfdb_find_recently_changed_files_t    find_recently_changed_files_op;
@@ -598,5 +584,3 @@ typedef struct gfdb_connection {
 
 
 #endif
-
-