From 1f594799f93b7b3a88de14681a55ee5de4ce0888 Mon Sep 17 00:00:00 2001 From: Joel Wright Date: Sun, 20 Oct 2013 19:36:19 +0200 Subject: [PATCH] Update tests and fix SizeFSAlphaNum generator bug when reading >64*1024 bytes --- README.md | 99 +++++++++++++++++++++------------------------ sizefs/__init__.py | 4 +- sizefs/contents.pyx | 13 +++++- sizefs/sizefs.py | 19 ++++++--- tests/test_sfs.py | 33 ++++++++------- 5 files changed, 92 insertions(+), 76 deletions(-) diff --git a/README.md b/README.md index 2261c35..874f179 100644 --- a/README.md +++ b/README.md @@ -42,91 +42,86 @@ Create Size File objects in memory: from sizefs import SizeFS sfs = SizeFS() - sfs.read('/1B', 1, 0, None) - sfs.read('/2B', 2, 0, None) - sfs.read('/2K', 1024, 0, None) - sfs.read('/128K', 1024*128, 0, None) - sfs.read('/4G', 4*1024*1024, 0, None) + sfs.read('/ones/1B', 1, 0, None, create=True) + sfs.read('/ones/2B', 2, 0, None, create=True) + sfs.read('/ones/2K', 1024, 0, None, create=True) + sfs.read('/ones/128K', 1024*128, 0, None, create=True) + sfs.read('/ones/4G', 4*1024*1024, 0, None, create=True) The folder structure is used to determine the content of the files: - sfs.read('/zeros/5B', 5, 0, None).read(0, 5) + sfs.read('/zeros/5B', 5, 0, None, create=True).read(0, 5) out> 00000 - sfs.read('/ones/5B', 5, 0, None).read(0, 5) + sfs.read('/ones/5B', 5, 0, None, create=True).read(0, 5) out> 11111 - sfs.read('/alpha_num/5B', 5, 0, None).read(0, 5) + sfs.read('/alpha_num/5B', 5, 0, None, create=True).read(0, 5) out> TMdEv -Folders can be created to manipulate the data: +The folders 'ones', 'zeros' and 'alpha_num' are always present, +but new folders can also be created. When files are created in a +folder, the xattrs of the folder determine that file's content until +the file's xattrs are updated: sfs.mkdir('/regex1', None) - sfs.setxattr('/regex1', 'filler', '0', None) - print sfs.read('/alpha_num/5B', 5, 0, None).read(0, 5) + sfs.setxattr('/regex1', 'generator', 'regex', None) + sfs.setxattr('/regex1', 'filler', 'regex', None) + print sfs.read('/regex1/5B', 5, 0, None, create=True).read(0, 5) - out> 00000 + out> regex - sfs.mkdir('/regex2', None) - sfs.setxattr('/regex2', 'filler', '1', None) - print sfs.read('/regex2/5B', 5, 0, None).read(0, 5) + sfs.setxattr('/regex1/5B', 'filler', 'string', None) + print sfs.read('/regex1/5B', 5, 0, None).read(0, 5) - out> 11111 + out> string - sfs.mkdir('/regex3', None) - sfs.setxattr('/regex3', 'filler', '[a-zA-Z0-9]', None) - print sfs.read('/regex3/5B', 5, 0, None).read(0, 5) + sfs.setxattr('/regex1/5B', 'filler', 'a{2}b{2}c', None) + print sfs.read('/regex1/5B', 5, 0, None).read(0, 5) - out> 1JAbd + out> aabbc -Files can be added to SizeFS using sfs.create: +Files can also be added to SizeFS using sfs.create: - sfs.mkdir('/regex3', None) - sfs.setxattr('/regex3', 'filler', '[a-zA-Z0-9]', None) - sfs.create('/regex3/5B', None) - print sfs.read('/regex3/5B', 5, 0, None) + sfs.mkdir('/folder', None) + sfs.create('/folder/5B', None) + print sfs.read('/folder/5B', 5, 0, None) + + out> 11111 - out> aS8yG +And as discussed above, the name of the file determines its size: sfs.create('/regex3/128K', None) - print len(sfs.read('/regex3/128K', 128*1024, 0, None)) + # Try to read more contents than the files contains + print len(sfs.read('/regex3/128K', 256*1000, 0, None)) - out> 131072 + out> 128000 sfs.create('/regex3/128K-1B', None) - print len(sfs.read('/regex3/128K-1B', 128*1024, 0, None)) + # Try to read more contents than the files contains + print len(sfs.read('/regex3/128K-1B', 256*1000, 0, None)) - out> 131071 + out> 127999 sfs.create('/regex3/128K+1B', None) - print len(sfs.read('/alphanum/128K+1B', 128*1024+1, 0, None)) - - out> 131073 - -File content can be generated that matches a regex pattern by adding a directory - - sfs.mkdir('/regex1') - sfs.setxattr('/regex1','filler','a(bcd)*e{4}[a-z03]*') - sfs.create('/regex1','128K') - print len(sfs.open('regex1/128KB').read(0, 128*1024)) + # Try to read more contents than the files contains + print len(sfs.read('/alphanum/128K+1B', 256*1000, 0, None)) - out> 131072 - - sfs.create('/regex1','128K-1B') - print len(sfs.open('regex1/128K-1B').read(0, 128*1024-1)) - - out> 131071 - - sfs.create('/regex1','128K+1B') - print len(sfs.open('regex1/128KB+1B').read(0, 128*1024+1)) - - out> 131073 + out> 128001 Extended Usage -------------- -We can set up to 5 properties: +The 'generator' xattr property defines the file content and can be set to one +of: + + ones - files are filled with ones + zeros - files are filled with zeros + alpha_num - files are filled with alpha numeric characters + regex - files are filled according to a collection of regular expression patterns + +We can set up to 5 properties to control the regular expression patterns: prefix - defined pattern for the start of a file (default = "") suffix - defined pattern for the end of a file (default = "") diff --git a/sizefs/__init__.py b/sizefs/__init__.py index cc0db54..0911c01 100644 --- a/sizefs/__init__.py +++ b/sizefs/__init__.py @@ -1,3 +1,3 @@ -__all__ = ['SizeFS', 'SizeFile'] +__all__ = ['SizeFS'] -from sizefs import SizeFS, SIZEFSCHARS +from sizefs import SizeFS diff --git a/sizefs/contents.pyx b/sizefs/contents.pyx index 05ddbed..c9ec82d 100644 --- a/sizefs/contents.pyx +++ b/sizefs/contents.pyx @@ -55,11 +55,20 @@ class SizeFSAlphaNumGen(object): def __init__(self): self.pre_seed = ''.join(random.choice(self.chars) - for _x in range(64*1024)) + for _ in range(64*1024)) def read(self, start, end): if start <= end: - return self.pre_seed[0:end-start+1] + toRead = end - start + pre_seed_count = 0 + content = [] + pre_seed_len = len(self.pre_seed) + while toRead > pre_seed_len: + content.append(self.pre_seed) + toRead -= pre_seed_len + pre_seed_count += 1 + content.append(self.pre_seed[0:toRead+1]) + return ''.join(content) else: return '' diff --git a/sizefs/sizefs.py b/sizefs/sizefs.py index 00724a9..b77e10a 100644 --- a/sizefs/sizefs.py +++ b/sizefs/sizefs.py @@ -169,7 +169,7 @@ def getattr(self, path, fh=None): if parent_folder in self.folders: return self.folders[parent_folder] - raise FuseOSError(ENOENT) + raise FuseOSError(ENOENT) def getxattr(self, path, name, position=0): """ @@ -229,15 +229,24 @@ def open(self, path, flags): self.fd += 1 return self.fd - def read(self, path, size, offset, fh): + def read(self, path, size, offset, fh, create=False): """ Returns content based on the pattern of the containing folder """ if path in self.files: - content = self.files[path]['generator'].read(offset, offset+size-1) - return content + size_bytes = self.files[path]['attrs']['st_size'] + if offset > (size_bytes - 1): + return "" + else: + end_of_content = min(offset+size-1, size_bytes-1) + content = self.files[path]['generator'].read(offset, end_of_content) + return content else: - raise FuseOSError(ENOENT) + if not create: + raise FuseOSError(ENOENT) + else: + self.create(path, 0444) + return self.read(path, size, offset, fh, create=create) def readdir(self, path, fh): contents = ['.', '..'] diff --git a/tests/test_sfs.py b/tests/test_sfs.py index ca53c8a..3f63d33 100644 --- a/tests/test_sfs.py +++ b/tests/test_sfs.py @@ -1,6 +1,7 @@ __author__ = 'jjw' -from sizefs import SizeFS, SIZEFSCHARS +from sizefs import SizeFS +from sizefs.contents import SizeFSAlphaNumGen import unittest @@ -10,30 +11,32 @@ def test_basic(self): # Basic Test sfs = SizeFS() - self.assertEqual(len(sfs.read('/zeros/1B', 1, 0, None)), 1) - self.assertEqual(len(sfs.read('/ones/1B', 1, 0, None)), 1) - self.assertEqual(len(sfs.read('/alpha_num/1B', 1, 0, None)), 1) + self.assertEqual(len(sfs.read('/zeros/1B', 1, 0, None, create=True)), 1) + self.assertEqual(len(sfs.read('/ones/1B', 1, 0, None, create=True)), 1) + self.assertEqual(len(sfs.read('/alpha_num/1B', 1, 0, None, create=True)), 1) def test_contents(self): # Contents Test sfs = SizeFS() - self.assertEqual(sfs.read('/zeros/5B', 5, 0, None), '00000') - self.assertEqual(sfs.read('/ones/5B', 5, 0, None), '11111') - for ch in sfs.read('/alpha_num/5B', 5, 0, None): - self.assertIn(ch, SIZEFSCHARS) + self.assertEqual(sfs.read('/zeros/5B', 5, 0, None, create=True), '00000') + self.assertEqual(sfs.read('/ones/5B', 5, 0, None, create=True), '11111') + for ch in sfs.read('/alpha_num/5B', 5, 0, None, create=True): + self.assertIn(ch, SizeFSAlphaNumGen.chars) def test_length(self): # Length Test k128 = 128*1000 + k256 = 256*1000 sfs = SizeFS() - self.assertEqual(len(sfs.read('/128B', 128, 0, None)), 128) - self.assertEqual(len(sfs.read('/128K', k128-1, 0, None)), k128-1) - self.assertEqual(len(sfs.read('/128K', k128, 0, None)), k128) - self.assertEqual(len(sfs.read('/128K+1B', k128+1, 0, None)), k128+1) - self.assertEqual(len(sfs.read('/zeros/5B', 5, 0, None)), 5) - self.assertEqual(len(sfs.read('/ones/5B', 5, 0, None)), 5) - self.assertEqual(len(sfs.read('/alpha_num/5B', 5, 0, None)), 5) + self.assertEqual(len(sfs.read('/zeros/128B', 128, 0, None, create=True)), 128) + self.assertEqual(len(sfs.read('/zeros/128K', k128-1, 0, None, create=True)), k128-1) + self.assertEqual(len(sfs.read('/alpha_num/128K', k128, 0, None, create=True)), k128) + self.assertEqual(len(sfs.read('/zeros/128K+1B', k128+1, 0, None, create=True)), k128+1) + self.assertEqual(len(sfs.read('/zeros/128K', k256, 0, None, create=True)), k128) + self.assertEqual(len(sfs.read('/zeros/5B', 5, 0, None, create=True)), 5) + self.assertEqual(len(sfs.read('/ones/5B', 5, 0, None, create=True)), 5) + self.assertEqual(len(sfs.read('/alpha_num/5B', 5, 0, None, create=True)), 5) if __name__ == '__main__':