diff options
| -rw-r--r-- | geo-replication/syncdaemon/gsyncd.py | 7 | ||||
| -rw-r--r-- | geo-replication/syncdaemon/master.py | 10 |
2 files changed, 8 insertions, 9 deletions
diff --git a/geo-replication/syncdaemon/gsyncd.py b/geo-replication/syncdaemon/gsyncd.py index 45b5ff546fa..3ddcb7f5454 100644 --- a/geo-replication/syncdaemon/gsyncd.py +++ b/geo-replication/syncdaemon/gsyncd.py @@ -278,6 +278,13 @@ def main_i(): op.add_option('--log-rsync-performance', default=False, action='store_true') op.add_option('--max-rsync-retries', type=int, default=10) + # Max size of Changelogs to process per batch, Changelogs Processing is + # not limited by the number of changelogs but instead based on + # size of the changelog file, One sample changelog file size was 145408 + # with ~1000 CREATE and ~1000 DATA. 5 such files in one batch is 727040 + # If geo-rep worker crashes while processing a batch, it has to retry only + # that batch since stime will get updated after each batch. + op.add_option('--changelog-batch-size', type=int, default=727040) op.add_option('--pause-on-start', default=False, action='store_true') op.add_option('-L', '--log-level', metavar='LVL') op.add_option('-r', '--remote-gsyncd', metavar='CMD', diff --git a/geo-replication/syncdaemon/master.py b/geo-replication/syncdaemon/master.py index b096bc77ebe..6365df8c532 100644 --- a/geo-replication/syncdaemon/master.py +++ b/geo-replication/syncdaemon/master.py @@ -37,14 +37,6 @@ URXTIME = (-1, 0) # crawl before starting live changelog crawl. CHANGELOG_ROLLOVER_TIME = 15 -# Max size of Changelogs to process per batch, Changelogs Processing is -# not limited by the number of changelogs but instead based on -# size of the changelog file, One sample changelog file size was 145408 -# with ~1000 CREATE and ~1000 DATA. 5 such files in one batch is 727040 -# If geo-rep worker crashes while processing a batch, it has to retry only -# that batch since stime will get updated after each batch. -MAX_CHANGELOG_BATCH_SIZE = 727040 - # Utility functions to help us to get to closer proximity # of the DRY principle (no, don't look for elevated or # perspectivistic things here) @@ -1166,7 +1158,7 @@ class GMasterChangelogMixin(GMasterCommon): current_size = 0 for c in changes: si = os.lstat(c).st_size - if (si + current_size) > MAX_CHANGELOG_BATCH_SIZE: + if (si + current_size) > int(gconf.changelog_batch_size): # Create new batch if single Changelog file greater than # Max Size! or current batch size exceeds Max size changelogs_batches.append([c]) |