From 601884e6e861813e14f9064e2f135eb857d59b17 Mon Sep 17 00:00:00 2001 From: Prashanth Pai Date: Tue, 31 May 2016 18:59:56 +0530 Subject: Implement shutil.copy* methods and os.link() Change-Id: I2de796e7d53732c5a967c6194a43378171fcb3d6 Signed-off-by: Prashanth Pai --- gluster/api.py | 5 + gluster/exceptions.py | 4 + gluster/gfapi.py | 147 ++++++++++++++++++++++- test/functional/libgfapi-python-tests.py | 197 ++++++++++++++++++++++++++++++- 4 files changed, 351 insertions(+), 2 deletions(-) diff --git a/gluster/api.py b/gluster/api.py index 3fd9d91..bb0d31f 100755 --- a/gluster/api.py +++ b/gluster/api.py @@ -402,6 +402,11 @@ glfs_rename = gfapi_prototype('glfs_rename', ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p) +glfs_link = gfapi_prototype('glfs_link', ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_char_p) + glfs_symlink = gfapi_prototype('glfs_symlink', ctypes.c_int, ctypes.c_void_p, ctypes.c_char_p, diff --git a/gluster/exceptions.py b/gluster/exceptions.py index 962e69f..b95e543 100644 --- a/gluster/exceptions.py +++ b/gluster/exceptions.py @@ -20,3 +20,7 @@ class LibgfapiException(Exception): class VolumeNotMounted(LibgfapiException): pass + + +class Error(EnvironmentError): + pass diff --git a/gluster/gfapi.py b/gluster/gfapi.py index 121d442..3d65155 100755 --- a/gluster/gfapi.py +++ b/gluster/gfapi.py @@ -16,7 +16,7 @@ import time import stat import errno from gluster import api -from gluster.exceptions import LibgfapiException +from gluster.exceptions import LibgfapiException, Error from gluster.utils import validate_mount, validate_glfd # TODO: Move this utils.py @@ -1210,6 +1210,18 @@ class Volume(object): raise OSError(err, os.strerror(err)) return s + @validate_mount + def link(self, source, link_name): + """ + Create a hard link pointing to source named link_name. + + :raises: OSError on failure + """ + ret = api.glfs_link(self.fs, source, link_name) + if ret < 0: + err = ctypes.get_errno() + raise OSError(err, os.strerror(err)) + @validate_mount def symlink(self, source, link_name): """ @@ -1307,3 +1319,136 @@ class Volume(object): yield x if not topdown: yield top, dirs, nondirs + + def samefile(self, path1, path2): + """ + Return True if both pathname arguments refer to the same file or + directory (as indicated by device number and inode number). Raise an + exception if a stat() call on either pathname fails. + + :param path1: Path to one file + :param path2: Path to another file + :raises: OSError if stat() fails + """ + s1 = self.stat(path1) + s2 = self.stat(path2) + return s1.st_ino == s2.st_ino and s1.st_dev == s2.st_dev + + @classmethod + def copyfileobj(self, fsrc, fdst, length=128 * 1024): + """ + Copy the contents of the file-like object fsrc to the file-like object + fdst. The integer length, if given, is the buffer size. Note that if + the current file position of the fsrc object is not 0, only the + contents from the current file position to the end of the file will be + copied. + + :param fsrc: Source file object + :param fdst: Destination file object + :param length: Size of buffer in bytes to be used in copying + :raises: OSError on failure + """ + buf = bytearray(length) + while True: + nread = fsrc.readinto(buf) + if not nread or nread <= 0: + break + if nread == length: + # Entire buffer is filled, do not slice. + fdst.write(buf) + else: + # TODO: + # Use memoryview to avoid internal copy done on slicing. + fdst.write(buf[0:nread]) + + def copyfile(self, src, dst): + """ + Copy the contents (no metadata) of the file named src to a file named + dst. dst must be the complete target file name. If src and dst are + the same, Error is raised. The destination location must be writable. + If dst already exists, it will be replaced. Special files such as + character or block devices and pipes cannot be copied with this + function. src and dst are path names given as strings. + + :param src: Path of source file + :param dst: Path of destination file + :raises: Error if src and dst file are same file. + OSError on failure to read/write. + """ + _samefile = False + try: + _samefile = self.samefile(src, dst) + except OSError: + # Dst file need not exist. + pass + + if _samefile: + raise Error("`%s` and `%s` are the same file" % (src, dst)) + + with self.fopen(src, 'rb') as fsrc: + with self.fopen(dst, 'wb') as fdst: + self.copyfileobj(fsrc, fdst) + + def copymode(self, src, dst): + """ + Copy the permission bits from src to dst. The file contents, owner, + and group are unaffected. src and dst are path names given as strings. + + :param src: Path of source file + :param dst: Path of destination file + :raises: OSError on failure. + """ + st = self.stat(src) + mode = stat.S_IMODE(st.st_mode) + self.chmod(dst, mode) + + def copystat(self, src, dst): + """ + Copy the permission bits, last access time, last modification time, + and flags from src to dst. The file contents, owner, and group are + unaffected. src and dst are path names given as strings. + + :param src: Path of source file + :param dst: Path of destination file + :raises: OSError on failure. + """ + st = self.stat(src) + mode = stat.S_IMODE(st.st_mode) + self.utime(dst, (st.st_atime, st.st_mtime)) + self.chmod(dst, mode) + # TODO: Handle st_flags on FreeBSD + + def copy(self, src, dst): + """ + Copy data and mode bits ("cp src dst") + + Copy the file src to the file or directory dst. If dst is a directory, + a file with the same basename as src is created (or overwritten) in + the directory specified. Permission bits are copied. src and dst are + path names given as strings. + + :param src: Path of source file + :param dst: Path of destination file or directory + :raises: OSError on failure + """ + if self.isdir(dst): + dst = os.path.join(dst, os.path.basename(src)) + self.copyfile(src, dst) + self.copymode(src, dst) + + def copy2(self, src, dst): + """ + Similar to copy(), but metadata is copied as well - in fact, this is + just copy() followed by copystat(). This is similar to the Unix command + cp -p. + + The destination may be a directory. + + :param src: Path of source file + :param dst: Path of destination file or directory + :raises: OSError on failure + """ + if self.isdir(dst): + dst = os.path.join(dst, os.path.basename(src)) + self.copyfile(src, dst) + self.copystat(src, dst) diff --git a/test/functional/libgfapi-python-tests.py b/test/functional/libgfapi-python-tests.py index 4e339e0..8c422e6 100644 --- a/test/functional/libgfapi-python-tests.py +++ b/test/functional/libgfapi-python-tests.py @@ -13,10 +13,11 @@ import unittest import os import types import errno +import hashlib import threading from gluster.gfapi import File, Volume -from gluster.exceptions import LibgfapiException +from gluster.exceptions import LibgfapiException, Error from test import get_test_config from ConfigParser import NoSectionError, NoOptionError from uuid import uuid4 @@ -600,6 +601,200 @@ class FileOpsTest(unittest.TestCase): with File(self.vol.open(file_name, os.O_RDONLY)) as f: self.assertRaises(TypeError, f.readinto, str("buf")) + def test_link(self): + name1 = uuid4().hex + self.vol.fopen(name1, 'w').close() + name2 = uuid4().hex + self.vol.link(name1, name2) + self.assertTrue(self.vol.samefile(name1, name2)) + self.assertEqual(self.vol.stat(name1).st_nlink, 2) + self.assertEqual(self.vol.stat(name2).st_nlink, 2) + + # source does not exist + self.assertRaises(OSError, self.vol.link, 'nonexistent file', 'link') + # target already exists + self.assertRaises(OSError, self.vol.link, name1, name2) + + def test_copyfileobj(self): + # Create source file. + src_file = uuid4().hex + with self.vol.fopen(src_file, 'wb') as f: + for i in xrange(2): + f.write(os.urandom(128 * 1024)) + f.write(os.urandom(25 * 1024)) + # Change/set atime and mtime + (atime, mtime) = (692884800, 692884800) + self.vol.utime(src_file, (atime, mtime)) + + # Calculate checksum of source file. + src_file_checksum = hashlib.md5() + with self.vol.fopen(src_file, 'rb') as f: + src_file_checksum.update(f.read(32 * 1024)) + + # Copy file + dest_file = uuid4().hex + with self.vol.fopen(src_file, 'rb') as fsrc: + with self.vol.fopen(dest_file, 'wb') as fdst: + self.vol.copyfileobj(fsrc, fdst) + + # Calculate checksum of destination + dest_file_checksum = hashlib.md5() + with self.vol.fopen(dest_file, 'rb') as f: + dest_file_checksum.update(f.read(32 * 1024)) + + self.assertEqual(src_file_checksum.hexdigest(), + dest_file_checksum.hexdigest()) + + # Copy file with different buffer size + self.vol.unlink(dest_file) + with self.vol.fopen(src_file, 'rb') as fsrc: + with self.vol.fopen(dest_file, 'wb') as fdst: + self.vol.copyfileobj(fsrc, fdst, 32 * 1024) + + # Calculate checksum of destination + dest_file_checksum = hashlib.md5() + with self.vol.fopen(dest_file, 'rb') as f: + dest_file_checksum.update(f.read(32 * 1024)) + + self.assertEqual(src_file_checksum.hexdigest(), + dest_file_checksum.hexdigest()) + + # The destination file should not have same atime and mtime + src_stat = self.vol.stat(src_file) + dest_stat = self.vol.stat(dest_file) + self.assertNotEqual(src_stat.st_atime, dest_stat.st_atime) + self.assertNotEqual(src_stat.st_mtime, dest_stat.st_mtime) + + # Test over-writing destination that exists + dest_file = uuid4().hex + with self.vol.fopen(dest_file, 'w') as f: + data = "A boy wants this test to not fail." + f.write(data) + with self.vol.fopen(src_file, 'rb') as fsrc: + with self.vol.fopen(dest_file, 'wb') as fdst: + self.vol.copyfileobj(fsrc, fdst) + self.assertNotEqual(self.vol.stat(src_file).st_size, len(data)) + + # Test one of the file object is closed + f1 = self.vol.fopen(src_file, 'rb') + f1.close() + f2 = self.vol.fopen(dest_file, 'wb') + self.assertRaises(OSError, self.vol.copyfileobj, f1, f2) + f2.close() + + def test_copyfile_samefile(self): + # Source and destination same error + name = uuid4().hex + self.vol.fopen(name, 'w').close() + self.assertRaises(Error, self.vol.copyfile, name, name) + # Harlink test + name2 = uuid4().hex + self.vol.link(name, name2) + self.assertRaises(Error, self.vol.copyfile, name, name2) + + def test_copymode(self): + src_file = uuid4().hex + self.vol.fopen(src_file, 'w').close() + self.vol.chmod(src_file, 0644) + + dest_file = uuid4().hex + self.vol.fopen(dest_file, 'w').close() + self.vol.chmod(dest_file, 0640) + + self.vol.copymode(src_file, dest_file) + self.assertEqual(self.vol.stat(src_file).st_mode, + self.vol.stat(dest_file).st_mode) + + def test_copystat(self): + # Create source file and set mode, atime, mtime + src_file = uuid4().hex + self.vol.fopen(src_file, 'w').close() + self.vol.chmod(src_file, 0640) + (atime, mtime) = (692884800, 692884800) + self.vol.utime(src_file, (atime, mtime)) + + # Create destination file + dest_file = uuid4().hex + self.vol.fopen(dest_file, 'w').close() + + # Invoke copystat() + self.vol.copystat(src_file, dest_file) + + # Verify + src_stat = self.vol.stat(src_file) + dest_stat = self.vol.stat(dest_file) + self.assertEqual(src_stat.st_mode, dest_stat.st_mode) + self.assertEqual(src_stat.st_atime, dest_stat.st_atime) + self.assertEqual(src_stat.st_mtime, dest_stat.st_mtime) + + def test_copy(self): + # Create source file. + src_file = uuid4().hex + with self.vol.fopen(src_file, 'wb') as f: + for i in xrange(2): + f.write(os.urandom(128 * 1024)) + f.write(os.urandom(25 * 1024)) + + # Calculate checksum of source file. + src_file_checksum = hashlib.md5() + with self.vol.fopen(src_file, 'rb') as f: + src_file_checksum.update(f.read(32 * 1024)) + + # Copy file into dir + dest_dir = uuid4().hex + self.vol.mkdir(dest_dir) + self.vol.copy(src_file, dest_dir) + + # Calculate checksum of destination + dest_file = os.path.join(dest_dir, src_file) + dest_file_checksum = hashlib.md5() + with self.vol.fopen(dest_file, 'rb') as f: + dest_file_checksum.update(f.read(32 * 1024)) + + # verify data + self.assertEqual(src_file_checksum.hexdigest(), + dest_file_checksum.hexdigest()) + + # verify mode + src_stat = self.vol.stat(src_file) + dest_stat = self.vol.stat(dest_file) + self.assertEqual(src_stat.st_mode, dest_stat.st_mode) + + def test_copy2(self): + # Create source file. + src_file = uuid4().hex + with self.vol.fopen(src_file, 'wb') as f: + for i in xrange(2): + f.write(os.urandom(128 * 1024)) + f.write(os.urandom(25 * 1024)) + + # Calculate checksum of source file. + src_file_checksum = hashlib.md5() + with self.vol.fopen(src_file, 'rb') as f: + src_file_checksum.update(f.read(32 * 1024)) + + # Copy file into dir + dest_dir = uuid4().hex + self.vol.mkdir(dest_dir) + self.vol.copy(src_file, dest_dir) + + # Calculate checksum of destination + dest_file = os.path.join(dest_dir, src_file) + dest_file_checksum = hashlib.md5() + with self.vol.fopen(dest_file, 'rb') as f: + dest_file_checksum.update(f.read(32 * 1024)) + + # verify data + self.assertEqual(src_file_checksum.hexdigest(), + dest_file_checksum.hexdigest()) + + # verify mode and stat + src_stat = self.vol.stat(src_file) + dest_stat = self.vol.stat(dest_file) + self.assertEqual(src_stat.st_mode, dest_stat.st_mode) + self.assertEqual(src_stat.st_atime, dest_stat.st_atime) + self.assertEqual(src_stat.st_mtime, dest_stat.st_mtime) + class DirOpsTest(unittest.TestCase): -- cgit