From ac33dc6dbf1f982cf522556aa938ebfb0e6ddded Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Thu, 27 Aug 2015 11:36:19 +0530 Subject: Refactor read_metadata() method This change: * Simplifies read_metadata() method. * Validates pickle header before attempting to unpickle. This change does NOT fix the security vulnerability itself. That would be sent as a separate change. Change-Id: Id95bd584f3ad00fb075456544495f17f7038f991 Signed-off-by: Prashanth Pai Reviewed-by: Thiago da Silva Tested-by: Thiago da Silva Reviewed-on: http://review.gluster.org/13220 --- test/unit/common/test_diskdir.py | 8 ++++---- test/unit/common/test_utils.py | 33 ++++++++++++++++----------------- 2 files changed, 20 insertions(+), 21 deletions(-) (limited to 'test/unit/common') diff --git a/test/unit/common/test_diskdir.py b/test/unit/common/test_diskdir.py index ebc554a..3b95de8 100644 --- a/test/unit/common/test_diskdir.py +++ b/test/unit/common/test_diskdir.py @@ -18,7 +18,6 @@ import os import errno import tempfile -import cPickle as pickle import unittest import shutil import tarfile @@ -26,6 +25,7 @@ import hashlib from time import time from swift.common.utils import normalize_timestamp from gluster.swift.common import utils +from gluster.swift.common.utils import serialize_metadata, deserialize_metadata import gluster.swift.common.Glusterfs from test_utils import _initxattr, _destroyxattr, _setxattr, _getxattr from test.unit import FakeLogger @@ -283,7 +283,7 @@ class TestDiskCommon(unittest.TestCase): def test__dir_exists_read_metadata_exists(self): datadir = os.path.join(self.td, self.fake_drives[0]) fake_md = { "fake": (True,0) } - fake_md_p = pickle.dumps(fake_md, utils.PICKLE_PROTOCOL) + fake_md_p = serialize_metadata(fake_md) _setxattr(datadir, utils.METADATA_KEY, fake_md_p) dc = dd.DiskCommon(self.td, self.fake_drives[0], self.fake_accounts[0], self.fake_logger) @@ -339,7 +339,7 @@ class TestDiskCommon(unittest.TestCase): dc.update_metadata({'X-Container-Meta-foo': '42'}) assert 'X-Container-Meta-foo' in dc.metadata assert dc.metadata['X-Container-Meta-foo'] == '42' - md = pickle.loads(_getxattr(dc.datadir, utils.METADATA_KEY)) + md = deserialize_metadata(_getxattr(dc.datadir, utils.METADATA_KEY)) assert dc.metadata == md, "%r != %r" % (dc.metadata, md) del dc.metadata['X-Container-Meta-foo'] assert dc.metadata == md_copy @@ -1248,7 +1248,7 @@ class TestDiskAccount(unittest.TestCase): datadir = os.path.join(self.td, self.fake_drives[i]) fake_md = { "fake-drv-%d" % i: (True,0) } self.fake_md.append(fake_md) - fake_md_p = pickle.dumps(fake_md, utils.PICKLE_PROTOCOL) + fake_md_p = serialize_metadata(fake_md) _setxattr(datadir, utils.METADATA_KEY, fake_md_p) if i == 2: # Third drive has valid account metadata diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 55bd0cf..1fea1fc 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -19,7 +19,6 @@ import os import unittest import errno import xattr -import cPickle as pickle import tempfile import hashlib import tarfile @@ -27,6 +26,7 @@ import shutil from collections import defaultdict from mock import patch, Mock from gluster.swift.common import utils, Glusterfs +from gluster.swift.common.utils import deserialize_metadata, serialize_metadata from gluster.swift.common.exceptions import GlusterFileSystemOSError,\ GlusterFileSystemIOError from swift.common.exceptions import DiskFileNoSpace @@ -154,7 +154,7 @@ class TestUtils(unittest.TestCase): xkey = _xkey(path, utils.METADATA_KEY) assert len(_xattrs.keys()) == 1 assert xkey in _xattrs - assert orig_d == pickle.loads(_xattrs[xkey]) + assert orig_d == deserialize_metadata(_xattrs[xkey]) assert _xattr_op_cnt['set'] == 1 def test_write_metadata_err(self): @@ -205,13 +205,13 @@ class TestUtils(unittest.TestCase): assert xkey in _xattrs assert len(_xattrs[xkey]) <= utils.MAX_XATTR_SIZE payload += _xattrs[xkey] - assert orig_d == pickle.loads(payload) + assert orig_d == deserialize_metadata(payload) assert _xattr_op_cnt['set'] == 3, "%r" % _xattr_op_cnt def test_clean_metadata(self): path = "/tmp/foo/c" expected_d = {'a': 'y' * 150000} - expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) + expected_p = serialize_metadata(expected_d) for i in range(0, 3): xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] @@ -223,7 +223,7 @@ class TestUtils(unittest.TestCase): def test_clean_metadata_err(self): path = "/tmp/foo/c" xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps({'a': 'y'}, utils.PICKLE_PROTOCOL) + _xattrs[xkey] = serialize_metadata({'a': 'y'}) _xattr_rem_err[xkey] = errno.EOPNOTSUPP try: utils.clean_metadata(path) @@ -237,7 +237,7 @@ class TestUtils(unittest.TestCase): path = "/tmp/foo/r" expected_d = {'a': 'y'} xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) + _xattrs[xkey] = serialize_metadata(expected_d) res_d = utils.read_metadata(path) assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt @@ -252,7 +252,7 @@ class TestUtils(unittest.TestCase): path = "/tmp/foo/r" expected_d = {'a': 'y'} xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) + _xattrs[xkey] = serialize_metadata(expected_d) _xattr_get_err[xkey] = errno.EOPNOTSUPP try: utils.read_metadata(path) @@ -265,7 +265,7 @@ class TestUtils(unittest.TestCase): def test_read_metadata_multiple(self): path = "/tmp/foo/r" expected_d = {'a': 'y' * 150000} - expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) + expected_p = serialize_metadata(expected_d) for i in range(0, 3): xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] @@ -273,12 +273,12 @@ class TestUtils(unittest.TestCase): assert not expected_p res_d = utils.read_metadata(path) assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) - assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt + assert _xattr_op_cnt['get'] == 4, "%r" % _xattr_op_cnt def test_read_metadata_multiple_one_missing(self): path = "/tmp/foo/r" expected_d = {'a': 'y' * 150000} - expected_p = pickle.dumps(expected_d, utils.PICKLE_PROTOCOL) + expected_p = serialize_metadata(expected_d) for i in range(0, 2): xkey = _xkey(path, "%s%s" % (utils.METADATA_KEY, i or '')) _xattrs[xkey] = expected_p[:utils.MAX_XATTR_SIZE] @@ -287,7 +287,6 @@ class TestUtils(unittest.TestCase): res_d = utils.read_metadata(path) assert res_d == {} assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt - assert len(_xattrs.keys()) == 0, "Expected 0 keys, found %d" % len(_xattrs.keys()) def test_restore_metadata_none(self): # No initial metadata @@ -303,7 +302,7 @@ class TestUtils(unittest.TestCase): path = "/tmp/foo/i" initial_d = {'a': 'z'} xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps(initial_d, utils.PICKLE_PROTOCOL) + _xattrs[xkey] = serialize_metadata(initial_d) res_d = utils.restore_metadata(path, {'b': 'y'}) expected_d = {'a': 'z', 'b': 'y'} assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) @@ -315,7 +314,7 @@ class TestUtils(unittest.TestCase): path = "/tmp/foo/i" initial_d = {'a': 'z'} xkey = _xkey(path, utils.METADATA_KEY) - _xattrs[xkey] = pickle.dumps(initial_d, utils.PICKLE_PROTOCOL) + _xattrs[xkey] = serialize_metadata(initial_d) res_d = utils.restore_metadata(path, {}) expected_d = {'a': 'z'} assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d) @@ -420,7 +419,7 @@ class TestUtils(unittest.TestCase): assert xkey in _xattrs assert _xattr_op_cnt['get'] == 1 assert _xattr_op_cnt['set'] == 1 - md = pickle.loads(_xattrs[xkey]) + md = deserialize_metadata(_xattrs[xkey]) assert r_md == md for key in self.obj_keys: @@ -442,7 +441,7 @@ class TestUtils(unittest.TestCase): assert xkey in _xattrs assert _xattr_op_cnt['get'] == 1 assert _xattr_op_cnt['set'] == 1 - md = pickle.loads(_xattrs[xkey]) + md = deserialize_metadata(_xattrs[xkey]) assert r_md == md for key in self.obj_keys: @@ -515,7 +514,7 @@ class TestUtils(unittest.TestCase): assert xkey in _xattrs assert _xattr_op_cnt['get'] == 1 assert _xattr_op_cnt['set'] == 1 - md = pickle.loads(_xattrs[xkey]) + md = deserialize_metadata(_xattrs[xkey]) assert r_md == md for key in self.cont_keys: @@ -541,7 +540,7 @@ class TestUtils(unittest.TestCase): assert xkey in _xattrs assert _xattr_op_cnt['get'] == 1 assert _xattr_op_cnt['set'] == 1 - md = pickle.loads(_xattrs[xkey]) + md = deserialize_metadata(_xattrs[xkey]) assert r_md == md for key in self.acct_keys: -- cgit