summaryrefslogtreecommitdiffstats
path: root/extras/quota/quota_fsck.py
blob: 2803c6ec7c674e39ac31b39b763657fc6fdb9513 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
#!/usr/bin/python
# The following script enables, Detecting, Reporting and Fixing
# anomalies in quota accounting. Run this script with -h option
# for further details.

'''
   Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.

   This file is licensed to you under your choice of the GNU Lesser
   General Public License, version 3 or any later version (LGPLv3 or
   later), or the GNU General Public License, version 2 (GPLv2), in all
   cases as published by the Free Software Foundation.
'''
from __future__ import print_function
import os, sys, re
from stat import *
import subprocess
import argparse
import xattr

aggr_size = {}
verbose_mode = False
mnt_path = None
brick_path = None
obj_fix_count = 0
file_count = 0
dir_count = 0

#CONSTANTS
KB = 1024
MB = 1048576
GB = 1048576 * 1024
TB = 1048576 * 1048576

QUOTA_VERBOSE = 0
QUOTA_META_ABSENT = 1
QUOTA_SIZE_MISMATCH = 2

IS_DIRTY ='0x3100'
IS_CLEAN ='0x3000'


epilog_msg='''
            The script attempts to find any gluster accounting issues in the
            filesystem at the given subtree. The script crawls the given
            subdirectory tree doing a stat for all files and compares the
            size reported by gluster quota with the size reported by stat
            calls. Any mismatch is reported. In addition integrity of marker
            xattrs are verified.
            '''

def print_msg(log_type, path, xattr_dict = {}, stbuf = "", dir_size = None):
    if log_type == QUOTA_VERBOSE:
        print('%-24s %-60s\nxattr_values: %s\n%s\n' % {"Verbose", path,  xattr_dict, stbuf})
    elif log_type == QUOTA_META_ABSENT:
        print('%-24s %-60s\n%s\n' % {"Quota-Meta Absent", path, xattr_dict})
    elif log_type == QUOTA_SIZE_MISMATCH:
        print("mismatch")
        if dir_size is not None:
            print('%24s %60s %12s %12s' % {"Size Mismatch", path, xattr_dict['contri_size'],
                   dir_size})
        else:
            print('%-24s %-60s %-12i %-12i' % {"Size Mismatch", path, xattr_dict['contri_size'],
                   stbuf.st_size})

def size_differs_lot(s1, s2):
    '''
    There could be minor accounting differences between the stat based
    accounting and gluster accounting. To avoid these from throwing lot
    of false positives in our logs. using a threshold of 1M for now.
    TODO: For a deeply nested directory, at higher levels in hierarchy
    differences may not be significant, hence this check needs to be improved.
    '''
    if abs(s1-s2) > 0:
        return True
    else:
        return False

def fix_hardlink_accounting(curr_dict, accounted_dict, curr_size):
    '''
            Hard links are messy.. we have to account them for their parent
            directory. But, stop accounting at the most common ancestor.
            Eg:
                say we have 3 hardlinks : /d1/d2/h1, /d1/d3/h2 and /d1/h3

            suppose we encounter the hard links h1 first , then h2 and then h3.
            while accounting for h1, we account the size until root(d2->d1->/)
            while accounting for h2, we need to account only till d3. (as d1
            and / are accounted for this inode).
            while accounting for h3 we should not account at all.. as all
            its ancestors are already accounted for same inode.

            curr_dict                : dict of hardlinks that were seen and
                                       accounted by the current iteration.
            accounted_dict           : dict of hardlinks that has already been
                                       accounted for.

            size                     : size of the object as accounted by the
                                       curr_iteration.

            Return vale:
            curr_size                : size reduced by hardlink sizes for those
                                       hardlinks that has already been accounted
                                       in current subtree.
            Also delete the duplicate link from curr_dict.
    '''

    dual_accounted_links = set(curr_dict.keys()) & set(accounted_dict.keys())
    for link in dual_accounted_links:
        curr_size = curr_size - curr_dict[link]
        del curr_dict[link]
    return curr_size


def fix_xattr(file_name, mark_dirty):
    global obj_fix_count
    global mnt_path

    if mnt_path is None:
        return
    if mark_dirty:
        print("MARKING DIRTY: " + file_name)
        out = subprocess.check_output (["/usr/bin/setfattr", "-n",
                                       "trusted.glusterfs.quota.dirty",
                                       "-v", IS_DIRTY, file_name])
    rel_path = os.path.relpath(file_name, brick_path)
    print("stat on "  + mnt_path + "/" + rel_path)
    stbuf = os.lstat(mnt_path + "/" + rel_path)

    obj_fix_count += 1

def get_quota_xattr_brick(dpath):
    out = subprocess.check_output (["/usr/bin/getfattr", "--no-dereference",
                                    "-d", "-m.", "-e", "hex", dpath])
    pairs = out.splitlines()

    '''
    Sample output to be parsed:
    [root@dhcp35-100 mnt]# getfattr -d -m. -e hex /export/b1/B0/d14/d13/
    # file: export/b1/B0/d14/d13/
    security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a7573725f743a733000
    trusted.gfid=0xbae5e0d2d05043de9fd851d91ecf63e8
    trusted.glusterfs.dht=0x000000010000000000000000ffffffff
    trusted.glusterfs.dht.mds=0x00000000
    trusted.glusterfs.quota.6a7675a3-b85a-40c5-830b-de9229d702ce.contri.39=0x00000000000000000000000000000000000000000000000e
    trusted.glusterfs.quota.dirty=0x3000
    trusted.glusterfs.quota.size.39=0x00000000000000000000000000000000000000000000000e
    '''

    '''
    xattr_dict dictionary holds quota related xattrs
    eg:
    '''

    xattr_dict = {}
    xattr_dict['parents'] = {}

    for xattr in pairs:
        xattr_key = xattr.split("=")[0]
        if re.search("# file:", xattr_key):
            # skip the file comment
            continue
        elif xattr_key is "":
            # skip any empty lines
            continue
        elif not re.search("quota", xattr_key):
            # skip all non quota xattr.
            continue

        xattr_value = xattr.split("=")[1]
        if re.search("contri", xattr_key):

            xattr_version = xattr_key.split(".")[5]
            if 'version' not in xattr_dict:
                xattr_dict['version'] = xattr_version
            else:
                if xattr_version != xattr_dict['version']:
                   print("Multiple xattr version found")


            cur_parent = xattr_key.split(".")[3]
            if cur_parent not in xattr_dict['parents']:
                xattr_dict['parents'][cur_parent] = {}

            contri_dict = xattr_dict['parents'][cur_parent]
            if len(xattr_value) == 34:
                # 34 bytes implies file contri xattr
                # contri format =0x< 16bytes file size><16bytes file count>
                # size is obtained in iatt, file count = 1, dir count=0
                contri_dict['contri_size'] = int(xattr_value[2:18], 16)
                contri_dict['contri_file_count'] = int(xattr_value[18:34], 16)
                contri_dict['contri_dir_count'] = 0
            else:
                # This is a directory contri.
                contri_dict['contri_size'] = int(xattr_value[2:18], 16)
                contri_dict['contri_file_count'] = int(xattr_value[18:34], 16)
                contri_dict['contri_dir_count'] = int(xattr_value[34:], 16)

        elif re.search("size", xattr_key):
            xattr_dict['size'] = int(xattr_value[2:18], 16)
            xattr_dict['file_count'] = int(xattr_value[18:34], 16)
            xattr_dict['dir_count'] = int(xattr_value[34:], 16)
        elif re.search("dirty", xattr_key):
            if xattr_value == IS_CLEAN:
                xattr_dict['dirty'] = False
            elif xattr_value == IS_DIRTY:
                xattr_dict['dirty'] = True
        elif re.search("limit_objects", xattr_key):
            xattr_dict['limit_objects'] = int(xattr_value[2:18], 16)
        elif re.search("limit_set", xattr_key):
            xattr_dict['limit_set'] = int(xattr_value[2:18], 16)

    return xattr_dict

def verify_file_xattr(path, stbuf = None):

    global file_count
    file_count += 1

    if stbuf is None:
        stbuf = os.lstat(path)

    xattr_dict = get_quota_xattr_brick(path)

    for parent in xattr_dict['parents']:
        contri_dict = xattr_dict['parents'][parent]

        if 'contri_size' not in contri_dict or \
           'contri_file_count' not in contri_dict or \
           'contri_dir_count' not in contri_dict:
            print_msg(QUOTA_META_ABSENT, path, xattr_dict, stbuf)
            fix_xattr(path, False)
            return
        elif size_differs_lot(contri_dict['contri_size'], stbuf.st_size):
            print_msg(QUOTA_SIZE_MISMATCH, path, xattr_dict, stbuf)
            fix_xattr(path, False)
            return

    if verbose_mode is True:
        print_msg(QUOTA_VERBOSE, path, xattr_dict, stbuf)


def verify_dir_xattr(path, dir_size):

    global dir_count
    dir_count += 1
    xattr_dict = get_quota_xattr_brick(path)

    stbuf = os.lstat(path)

    for parent in xattr_dict['parents']:
        contri_dict = xattr_dict['parents'][parent]

        if 'size' not in xattr_dict or 'contri_size' not in contri_dict:
            print_msg(QUOTA_META_ABSENT, path)
            fix_xattr(path, True)
            return
        elif size_differs_lot(dir_size, xattr_dict['size']) or \
             size_differs_lot(contri_dict['contri_size'], xattr_dict['size']):
            print_msg(QUOTA_SIZE_MISMATCH, path, xattr_dict, stbuf, dir_size)
            fix_xattr(path, True)
            return

    if verbose_mode is True:
        print_msg("VERBOSE", path, xattr_dict, stbuf, dir_size)


def walktree(t_dir, hard_link_dict):
    '''recursively descend the directory tree rooted at dir,
       aggregating the size
       t_dir            : directory to walk over.
       hard_link_dict   : dict of inodes with multiple hard_links under t_dir
    '''
    global aggr_size
    aggr_size[t_dir] = 0

    for entry in os.listdir(t_dir):
        pathname = os.path.join(t_dir, entry)
        stbuf = os.lstat(pathname)
        if S_ISDIR(stbuf.st_mode):
            # It's a directory, recurse into it
            if entry == '.glusterfs':
                print("skipping " + pathname)
                continue
            descendent_hardlinks = {}
            subtree_size = walktree(pathname, descendent_hardlinks)

            subtree_size = fix_hardlink_accounting(descendent_hardlinks,
                                                   hard_link_dict,
                                                   subtree_size)

            aggr_size[t_dir] = aggr_size[t_dir] + subtree_size

        elif S_ISREG(stbuf.st_mode) or S_ISLNK(stbuf.st_mode):
            # Even a symbolic link file may have multiple hardlinks.

            file_size = stbuf.st_size
            if stbuf.st_nlink > 2:
                # send a single element dict to check if file is accounted.
                file_size = fix_hardlink_accounting({stbuf.st_ino:stbuf.st_size},
                                                    hard_link_dict,
                                                    stbuf.st_size)

                if file_size == 0:
                    print_msg("HARD_LINK (skipped)", pathname, "",
                                stbuf)
                else:
                    print_msg("HARD_LINK (accounted)", pathname, "",
                               stbuf)
                    hard_link_dict[stbuf.st_ino] = stbuf.st_size

            if t_dir in aggr_size:
                aggr_size[t_dir] = aggr_size[t_dir] + file_size
            else:
                aggr_size[t_dir] = file_size
            verify_file_xattr(pathname, stbuf)

        else:
            # Unknown file type, print a message
            print('Skipping %s, due to file mode' % (pathname))

    if t_dir not in aggr_size:
        aggr_size[t_dir] = 0

    verify_dir_xattr(t_dir, aggr_size[t_dir])
    # du also accounts for t_directory sizes
    # aggr_size[t_dir] += 4096

    #cleanup
    ret = aggr_size[t_dir]
    del aggr_size[t_dir]
    return ret


if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Diagnose quota accounting issues.', epilog=epilog_msg)
    parser.add_argument('brick_path', nargs=1,
                        help='The brick path (or any descendent sub-directory of brick path)',
                        )
    parser.add_argument('--full-logs', dest='verbose', action='store_true',
                   help='''
                         log all the xattr values and stat values reported
                         for analysis. [CAUTION: This can give lot of output
                         depending on FS depth. So one has to make sure enough
                         disk space exists if redirecting to file]
                        '''
                        )
    parser.add_argument('--fix-issues', metavar='mount_path', dest='mnt', action='store',
                   help='''
                         fix accounting issues where the xattr values disagree
                         with stat sizes reported by gluster. A mount is also
                         required for this option to be used.
                         [CAUTION: This will directly modify backend xattr]
                        '''
                        )
    parser.add_argument('--sub-dir', metavar='sub_dir', dest='sub_dir', action='store',
                   help='''
                         limit the crawling and accounting verification/correction
                         to a specific subdirectory.
                        '''
                        )

    args = parser.parse_args()
    verbose_mode = args.verbose
    brick_path = args.brick_path[0]
    sub_dir = args.sub_dir
    mnt_path = args.mnt
    hard_link_dict = {}
    if sub_dir is not None:
        walktree(os.path.join(brick_path, sub_dir), hard_link_dict)
    else:
        walktree(brick_path, hard_link_dict)

    print("Files verified : " + str(file_count))
    print("Directories verified : " + str(dir_count))
    if mnt_path is not None:
        print("Objects Fixed : " + str(obj_fix_count))