From 7f92252f94ef50e1a63b55ba0c75c83841ec9433 Mon Sep 17 00:00:00 2001 From: Thijs Triemstra Date: Sun, 22 May 2016 02:46:51 +0200 Subject: [PATCH 01/27] add django 1.10 --- .travis.yml | 3 +++ tox.ini | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 97b8f1b..e0d9361 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,13 +15,16 @@ env: - TOXENV=py27-django-17 - TOXENV=py27-django-18 - TOXENV=py27-django-19 + - TOXENV=py27-django-110 - TOXENV=py33-django-17 - TOXENV=py33-django-18 - TOXENV=py34-django-17 - TOXENV=py34-django-18 - TOXENV=py34-django-19 + - TOXENV=py34-django-110 - TOXENV=py35-django-18 - TOXENV=py35-django-19 + - TOXENV=py35-django-110 after_success: coveralls deploy: provider: pypi diff --git a/tox.ini b/tox.ini index 49d0b91..39f969d 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,7 @@ downloadcache = {distshare} args_are_paths = false envlist = + {py27,py34,py35}-django-110 {py27,py34,py35}-django-19 {py27,py33,py34,py35}-django-18 {py27,py33,py34}-django-17 @@ -20,5 +21,6 @@ whitelist_externals = make deps = django-17: Django>=1.7,<1.8 django-18: Django>=1.8,<1.9 - django-19: Django>=1.9,<2.0 + django-19: Django>=1.9,<1.10 + django-110: Django>=1.10,<2.0 -rtests/requirements.txt From a7c8e07c0a12cd6e1ce7cd2d2d5f18d7206bf6a6 Mon Sep 17 00:00:00 2001 From: Thijs Triemstra Date: Sun, 22 May 2016 02:59:13 +0200 Subject: [PATCH 02/27] use django's VERSION --- queued_storage/backends.py | 5 +++-- queued_storage/utils.py | 2 +- tox.ini | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/queued_storage/backends.py b/queued_storage/backends.py index f9f58c2..ef80657 100644 --- a/queued_storage/backends.py +++ b/queued_storage/backends.py @@ -1,14 +1,15 @@ import six +from django import VERSION from django.core.cache import cache from django.core.exceptions import ImproperlyConfigured from django.utils.functional import SimpleLazyObject from django.utils.http import urlquote from .conf import settings -from .utils import import_attribute, django_version +from .utils import import_attribute -if django_version()[1] >= 7: +if VERSION[1] >= 7: from django.utils.deconstruct import deconstructible diff --git a/queued_storage/utils.py b/queued_storage/utils.py index a57e9c6..0f1cfe1 100644 --- a/queued_storage/utils.py +++ b/queued_storage/utils.py @@ -25,4 +25,4 @@ def import_attribute(import_path=None, options=None): def django_version(): - return [int(x) for x in django.get_version().split('.')] + return django.VERSION diff --git a/tox.ini b/tox.ini index 39f969d..d078a23 100644 --- a/tox.ini +++ b/tox.ini @@ -22,5 +22,5 @@ deps = django-17: Django>=1.7,<1.8 django-18: Django>=1.8,<1.9 django-19: Django>=1.9,<1.10 - django-110: Django>=1.10,<2.0 + django-110: Django==1.10a1 -rtests/requirements.txt From ed1d0a0900a62f7d19b6afac68b536c80ed13f8a Mon Sep 17 00:00:00 2001 From: Thijs Triemstra Date: Sun, 22 May 2016 03:04:45 +0200 Subject: [PATCH 03/27] remove another django_version reference --- queued_storage/backends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queued_storage/backends.py b/queued_storage/backends.py index ef80657..00fb93a 100644 --- a/queued_storage/backends.py +++ b/queued_storage/backends.py @@ -335,7 +335,7 @@ def modified_time(self, name): :rtype: :class:`~python:datetime.datetime` """ return self.get_storage(name).modified_time(name) -if django_version()[1] >= 7: +if VERSION[1] >= 7: QueuedStorage = deconstructible(QueuedStorage) From 1a86ba05b8a966309fc66af4828f8be1336aa6fb Mon Sep 17 00:00:00 2001 From: Thijs Triemstra Date: Mon, 23 May 2016 01:10:52 +0200 Subject: [PATCH 04/27] implement generate_filename --- queued_storage/backends.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/queued_storage/backends.py b/queued_storage/backends.py index 00fb93a..276e65a 100644 --- a/queued_storage/backends.py +++ b/queued_storage/backends.py @@ -1,3 +1,5 @@ +import os + import six from django import VERSION @@ -166,7 +168,7 @@ def open(self, name, mode='rb'): """ return self.get_storage(name).open(name, mode) - def save(self, name, content): + def save(self, name, content, max_length=None): """ Saves the given content with the given name using the local storage. If the :attr:`~queued_storage.backends.QueuedStorage.delayed` @@ -178,6 +180,9 @@ def save(self, name, content): :type name: str :param content: content of the file specified by name :type content: :class:`~django:django.core.files.File` + :param max_length: The length of the filename will not exceed + `max_length`, if provided. + :type max_length: int :rtype: str """ cache_key = self.get_cache_key(name) @@ -185,7 +190,7 @@ def save(self, name, content): # Use a name that is available on both the local and remote storage # systems and save locally. - name = self.get_available_name(name) + name = self.get_available_name(name, max_length) name = self.local.save(name, content) # Pass on the cache key to prevent duplicate cache key creation, @@ -222,7 +227,7 @@ def get_valid_name(self, name): """ return self.get_storage(name).get_valid_name(name) - def get_available_name(self, name): + def get_available_name(self, name, max_length=None): """ Returns a filename that's free on both the local and remote storage systems, and available for new content to be written to. @@ -238,6 +243,15 @@ def get_available_name(self, name): return remote_available_name return local_available_name + def generate_filename(self, filename): + """ + Validate the filename by calling get_valid_name() and return a filename + to be passed to the save() method. + """ + # `filename` may include a path as returned by FileField.upload_to. + dirname, filename = os.path.split(filename) + return os.path.normpath(os.path.join(dirname, self.get_valid_name(filename))) + def path(self, name): """ Returns a local filesystem path where the file can be retrieved using From 12decb68e5f27f612c9f754fe3d651ab688c2c8a Mon Sep 17 00:00:00 2001 From: Thijs Triemstra Date: Wed, 22 Jun 2016 04:41:51 +0200 Subject: [PATCH 05/27] update to django 1.10b1 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index d078a23..7cc66c4 100644 --- a/tox.ini +++ b/tox.ini @@ -22,5 +22,5 @@ deps = django-17: Django>=1.7,<1.8 django-18: Django>=1.8,<1.9 django-19: Django>=1.9,<1.10 - django-110: Django==1.10a1 + django-110: Django==1.10b1 -rtests/requirements.txt From defcd578b17b69f3974d300774ba1491246a428c Mon Sep 17 00:00:00 2001 From: onekiloparsec Date: Fri, 26 May 2017 19:01:13 +0200 Subject: [PATCH 06/27] Propagating max_length parameter Signed-off-by: onekiloparsec --- queued_storage/backends.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/queued_storage/backends.py b/queued_storage/backends.py index 445b49d..0fcffde 100644 --- a/queued_storage/backends.py +++ b/queued_storage/backends.py @@ -236,8 +236,8 @@ def get_available_name(self, name, max_length=None): :type name: str :rtype: str """ - local_available_name = self.local.get_available_name(name) - remote_available_name = self.remote.get_available_name(name) + local_available_name = self.local.get_available_name(name, max_length=max_length) + remote_available_name = self.remote.get_available_name(name, max_length=max_length) if remote_available_name > local_available_name: return remote_available_name From e39176e189303f6a922c35c330339ea5d00bbe35 Mon Sep 17 00:00:00 2001 From: onekiloparsec Date: Fri, 26 May 2017 19:03:26 +0200 Subject: [PATCH 07/27] Applying suggestion to fix StorageTests Since Django 1.10 no temp files outside MEDIA_ROOT are allowed. Signed-off-by: onekiloparsec --- tests/test_storages.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_storages.py b/tests/test_storages.py index fa4679a..0519081 100644 --- a/tests/test_storages.py +++ b/tests/test_storages.py @@ -13,6 +13,7 @@ from django.core.files.base import File from django.core.files.storage import FileSystemStorage, Storage +from django.conf import settings as django_settings from django.test import TestCase from queued_storage.backends import QueuedStorage @@ -27,8 +28,8 @@ def setUp(self): self.old_celery_always_eager = getattr( settings, 'CELERY_ALWAYS_EAGER', False) settings.CELERY_ALWAYS_EAGER = True - self.local_dir = tempfile.mkdtemp() - self.remote_dir = tempfile.mkdtemp() + self.local_dir = tempfile.mkdtemp(dir=os.path.join(django_settings.MEDIA_ROOT, 'storage_tests_local')) + self.remote_dir = tempfile.mkdtemp(dir=os.path.join(django_settings.MEDIA_ROOT, 'storage_tests_local')) tmp_dir = tempfile.mkdtemp() self.test_file_name = 'queued_storage.txt' self.test_file_path = path.join(tmp_dir, self.test_file_name) From db3e97f14d8258e7818d28358b300c3427a99bee Mon Sep 17 00:00:00 2001 From: onekiloparsec Date: Fri, 26 May 2017 19:33:08 +0200 Subject: [PATCH 08/27] Okay, MEDIA_ROOT undefined, better that way. Signed-off-by: onekiloparsec --- tests/test_storages.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_storages.py b/tests/test_storages.py index 0519081..7d3a725 100644 --- a/tests/test_storages.py +++ b/tests/test_storages.py @@ -28,14 +28,19 @@ def setUp(self): self.old_celery_always_eager = getattr( settings, 'CELERY_ALWAYS_EAGER', False) settings.CELERY_ALWAYS_EAGER = True - self.local_dir = tempfile.mkdtemp(dir=os.path.join(django_settings.MEDIA_ROOT, 'storage_tests_local')) - self.remote_dir = tempfile.mkdtemp(dir=os.path.join(django_settings.MEDIA_ROOT, 'storage_tests_local')) tmp_dir = tempfile.mkdtemp() self.test_file_name = 'queued_storage.txt' self.test_file_path = path.join(tmp_dir, self.test_file_name) with open(self.test_file_path, 'a') as test_file: test_file.write('test') self.test_file = open(self.test_file_path, 'r') + + # Overriding MEDIA_ROOT for tests (probbaly undefined at that stage anyway) + # Starting with Django>=1.10, Storages APIs don't allow to write outside MEDIA_ROOT + django_settings.MEDIA_ROOT = tempfile.mkdtemp() + self.local_dir = os.path.join(django_settings.MEDIA_ROOT, 'storage_tests_local') + self.remote_dir = os.path.join(django_settings.MEDIA_ROOT, 'storage_tests_remote') + self.addCleanup(shutil.rmtree, self.local_dir) self.addCleanup(shutil.rmtree, self.remote_dir) self.addCleanup(shutil.rmtree, tmp_dir) From 868dcdddbb05337ce693337446120b4c4c13cb22 Mon Sep 17 00:00:00 2001 From: Thijs Triemstra Date: Fri, 26 May 2017 20:13:52 +0200 Subject: [PATCH 09/27] test against 1.11 --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 7cc66c4..d6c80c2 100644 --- a/tox.ini +++ b/tox.ini @@ -2,8 +2,7 @@ downloadcache = {distshare} args_are_paths = false envlist = - {py27,py34,py35}-django-110 - {py27,py34,py35}-django-19 + {py27,py34,py35}-django-{19,110,111} {py27,py33,py34,py35}-django-18 {py27,py33,py34}-django-17 @@ -22,5 +21,6 @@ deps = django-17: Django>=1.7,<1.8 django-18: Django>=1.8,<1.9 django-19: Django>=1.9,<1.10 - django-110: Django==1.10b1 + django-110: Django>=1.10,<1.11 + django-111: Django>=1.11,<1.12 -rtests/requirements.txt From 59ca6ae8fe7241a9abd25bce90dbb35e617ba3b1 Mon Sep 17 00:00:00 2001 From: Thijs Triemstra Date: Fri, 26 May 2017 20:17:11 +0200 Subject: [PATCH 10/27] test against 1.11 --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index e0d9361..5103f3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,15 +16,18 @@ env: - TOXENV=py27-django-18 - TOXENV=py27-django-19 - TOXENV=py27-django-110 + - TOXENV=py27-django-111 - TOXENV=py33-django-17 - TOXENV=py33-django-18 - TOXENV=py34-django-17 - TOXENV=py34-django-18 - TOXENV=py34-django-19 - TOXENV=py34-django-110 + - TOXENV=py35-django-111 - TOXENV=py35-django-18 - TOXENV=py35-django-19 - TOXENV=py35-django-110 + - TOXENV=py35-django-111 after_success: coveralls deploy: provider: pypi From 5be4f17bb2b527fdbef1ae580352ce3b62ec87d4 Mon Sep 17 00:00:00 2001 From: onekiloparsec Date: Fri, 26 May 2017 20:44:36 +0200 Subject: [PATCH 11/27] One needs to create dirs? Signed-off-by: onekiloparsec --- tests/test_storages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_storages.py b/tests/test_storages.py index 7d3a725..1b27ffd 100644 --- a/tests/test_storages.py +++ b/tests/test_storages.py @@ -40,6 +40,8 @@ def setUp(self): django_settings.MEDIA_ROOT = tempfile.mkdtemp() self.local_dir = os.path.join(django_settings.MEDIA_ROOT, 'storage_tests_local') self.remote_dir = os.path.join(django_settings.MEDIA_ROOT, 'storage_tests_remote') + os.mkdir(self.local_dir) + os.mkdir(self.remote_dir) self.addCleanup(shutil.rmtree, self.local_dir) self.addCleanup(shutil.rmtree, self.remote_dir) From 171482f23da2ec118de2bd234baa1a4594d4b0af Mon Sep 17 00:00:00 2001 From: onekiloparsec Date: Fri, 26 May 2017 21:12:45 +0200 Subject: [PATCH 12/27] Ok maybe not Signed-off-by: onekiloparsec --- queued_storage/backends.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/queued_storage/backends.py b/queued_storage/backends.py index 0fcffde..445b49d 100644 --- a/queued_storage/backends.py +++ b/queued_storage/backends.py @@ -236,8 +236,8 @@ def get_available_name(self, name, max_length=None): :type name: str :rtype: str """ - local_available_name = self.local.get_available_name(name, max_length=max_length) - remote_available_name = self.remote.get_available_name(name, max_length=max_length) + local_available_name = self.local.get_available_name(name) + remote_available_name = self.remote.get_available_name(name) if remote_available_name > local_available_name: return remote_available_name From 8416c627bb6dbd198a8eade2beea9b118552863c Mon Sep 17 00:00:00 2001 From: Ashley Camba Garrido Date: Fri, 2 Jun 2017 11:11:39 +0200 Subject: [PATCH 13/27] Probably fixes #28 by passing filename, content to storage save --- tests/test_storages.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/test_storages.py b/tests/test_storages.py index fa4679a..4c36bbf 100644 --- a/tests/test_storages.py +++ b/tests/test_storages.py @@ -94,7 +94,8 @@ def test_storage_simple_save(self): field = models.TestModel._meta.get_field('testfile') field.storage = storage - obj = models.TestModel(testfile=File(self.test_file)) + obj = models.TestModel() + obj.testfile.save(self.test_file_name, File(self.test_file)) obj.save() self.assertTrue(path.isfile(path.join(self.local_dir, obj.testfile.name))) @@ -113,9 +114,11 @@ def test_storage_celery_save(self): field = models.TestModel._meta.get_field('testfile') field.storage = storage - obj = models.TestModel(testfile=File(self.test_file)) + obj = models.TestModel() + obj.testfile.save(self.test_file_name, File(self.test_file)) obj.save() + self.assertTrue(obj.testfile.storage.result.get()) self.assertTrue(path.isfile(path.join(self.local_dir, obj.testfile.name))) self.assertTrue( @@ -165,7 +168,8 @@ def test_transfer_and_delete(self): field = models.TestModel._meta.get_field('testfile') field.storage = storage - obj = models.TestModel(testfile=File(self.test_file)) + obj = models.TestModel() + obj.testfile.save(self.test_file_name, File(self.test_file)) obj.save() obj.testfile.storage.result.get() @@ -192,7 +196,8 @@ def test_transfer_returns_boolean(self): field = models.TestModel._meta.get_field('testfile') field.storage = storage - obj = models.TestModel(testfile=File(self.test_file)) + obj = models.TestModel() + obj.testfile.save(self.test_file_name, File(self.test_file)) obj.save() self.assertRaises(ValueError, @@ -213,7 +218,8 @@ def test_transfer_retried(self): self.assertFalse(models.TestModel.retried) - obj = models.TestModel(testfile=File(self.test_file)) + obj = models.TestModel() + obj.testfile.save(self.test_file_name, File(self.test_file)) obj.save() self.assertFalse(obj.testfile.storage.result.get()) @@ -230,7 +236,8 @@ def test_delayed_storage(self): field = models.TestModel._meta.get_field('testfile') field.storage = storage - obj = models.TestModel(testfile=File(self.test_file)) + obj = models.TestModel() + obj.testfile.save(self.test_file_name, File(self.test_file)) obj.save() self.assertIsNone(getattr(obj.testfile.storage, 'result', None)) @@ -257,7 +264,8 @@ def test_remote_file_field(self): field = models.TestModel._meta.get_field('remote') field.storage = storage - obj = models.TestModel(remote=File(self.test_file)) + obj = models.TestModel() + obj.testfile.save(self.test_file_name, File(self.test_file)) obj.save() self.assertIsNone(getattr(obj.testfile.storage, 'result', None)) From f6016a35bb69f2b7539e8f43566e644ba55480b8 Mon Sep 17 00:00:00 2001 From: Ashley Camba Garrido Date: Fri, 2 Jun 2017 12:37:29 +0200 Subject: [PATCH 14/27] Update test settings --- .travis.yml | 25 +++++++------------------ setup.cfg | 2 +- tox.ini | 13 +++++++------ 3 files changed, 15 insertions(+), 25 deletions(-) diff --git a/.travis.yml b/.travis.yml index 97b8f1b..cdab2d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,16 @@ sudo: false language: python -addons: - apt: - sources: - - deadsnakes - packages: - - python3.5 +python: + - "2.7" + - "3.3" + - "3.4" + - "3.5" + - "3.6" cache: directories: - $HOME/.cache/pip -install: pip install tox coveralls wheel +install: pip install tox coveralls wheel tox-travis script: tox -env: - - TOXENV=py27-django-17 - - TOXENV=py27-django-18 - - TOXENV=py27-django-19 - - TOXENV=py33-django-17 - - TOXENV=py33-django-18 - - TOXENV=py34-django-17 - - TOXENV=py34-django-18 - - TOXENV=py34-django-19 - - TOXENV=py35-django-18 - - TOXENV=py35-django-19 after_success: coveralls deploy: provider: pypi diff --git a/setup.cfg b/setup.cfg index 568a748..5de337e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,4 @@ -[pytest] +[tool:pytest] django_find_project = false addopts = --cov queued_storage DJANGO_SETTINGS_MODULE = tests.settings diff --git a/tox.ini b/tox.ini index 49d0b91..52c8775 100644 --- a/tox.ini +++ b/tox.ini @@ -2,9 +2,9 @@ downloadcache = {distshare} args_are_paths = false envlist = - {py27,py34,py35}-django-19 - {py27,py33,py34,py35}-django-18 - {py27,py33,py34}-django-17 + py{27,34,35}-dj{19,110,111}, + py36-dj111 + [testenv] basepython = @@ -12,13 +12,14 @@ basepython = py33: python3.3 py34: python3.4 py35: python3.5 + py36: python3.6 usedevelop = true setenv = CELERY_CONFIG_MODULE=tests.celeryconfig commands = make test whitelist_externals = make deps = - django-17: Django>=1.7,<1.8 - django-18: Django>=1.8,<1.9 - django-19: Django>=1.9,<2.0 + dj19: Django>=1.9,<1.10 + dj110: Django>=1.10,<1.11 + dj111: Django>=1.11,<2.0 -rtests/requirements.txt From c30cc45153b9602465d1b54fb7e6d70be65745f7 Mon Sep 17 00:00:00 2001 From: Ashley Camba Garrido Date: Fri, 2 Jun 2017 13:31:50 +0200 Subject: [PATCH 15/27] Fixes #29 - adopt django 1.10 method names --- queued_storage/backends.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/queued_storage/backends.py b/queued_storage/backends.py index 445b49d..aaaef8b 100644 --- a/queued_storage/backends.py +++ b/queued_storage/backends.py @@ -349,6 +349,40 @@ def modified_time(self, name): :rtype: :class:`~python:datetime.datetime` """ return self.get_storage(name).modified_time(name) + + def get_accessed_time(self, name): + """ + Returns the last accessed time (as datetime object) of the file + specified by name. + + :param name: file name + :type name: str + :rtype: :class:`~python:datetime.datetime` + """ + return self.get_storage(name).accessed_time(name) + + def get_created_time(self, name): + """ + Returns the creation time (as datetime object) of the file + specified by name. + + :param name: file name + :type name: str + :rtype: :class:`~python:datetime.datetime` + """ + return self.get_storage(name).created_time(name) + + def get_modified_time(self, name): + """ + Returns the last modified time (as datetime object) of the file + specified by name. + + :param name: file name + :type name: str + :rtype: :class:`~python:datetime.datetime` + """ + return self.get_storage(name).modified_time(name) + if VERSION[1] >= 7: QueuedStorage = deconstructible(QueuedStorage) From 856a2ee8267a5c037d2e5b2534c157519d764bcc Mon Sep 17 00:00:00 2001 From: onekiloparsec Date: Wed, 19 Jul 2017 15:12:16 +0200 Subject: [PATCH 16/27] Increased tests versions and update tests setup Signed-off-by: onekiloparsec --- .travis.yml | 3 +++ setup.cfg | 2 +- tox.ini | 6 +++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index e0d9361..dda11ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,15 +16,18 @@ env: - TOXENV=py27-django-18 - TOXENV=py27-django-19 - TOXENV=py27-django-110 + - TOXENV=py27-django-111 - TOXENV=py33-django-17 - TOXENV=py33-django-18 - TOXENV=py34-django-17 - TOXENV=py34-django-18 - TOXENV=py34-django-19 - TOXENV=py34-django-110 + - TOXENV=py34-django-111 - TOXENV=py35-django-18 - TOXENV=py35-django-19 - TOXENV=py35-django-110 + - TOXENV=py35-django-111 after_success: coveralls deploy: provider: pypi diff --git a/setup.cfg b/setup.cfg index 568a748..5de337e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,4 @@ -[pytest] +[tool:pytest] django_find_project = false addopts = --cov queued_storage DJANGO_SETTINGS_MODULE = tests.settings diff --git a/tox.ini b/tox.ini index 7cc66c4..d6c80c2 100644 --- a/tox.ini +++ b/tox.ini @@ -2,8 +2,7 @@ downloadcache = {distshare} args_are_paths = false envlist = - {py27,py34,py35}-django-110 - {py27,py34,py35}-django-19 + {py27,py34,py35}-django-{19,110,111} {py27,py33,py34,py35}-django-18 {py27,py33,py34}-django-17 @@ -22,5 +21,6 @@ deps = django-17: Django>=1.7,<1.8 django-18: Django>=1.8,<1.9 django-19: Django>=1.9,<1.10 - django-110: Django==1.10b1 + django-110: Django>=1.10,<1.11 + django-111: Django>=1.11,<1.12 -rtests/requirements.txt From 30d12b1759951a4efa9b3db08e884d46b01308a0 Mon Sep 17 00:00:00 2001 From: onekiloparsec Date: Wed, 19 Jul 2017 16:45:50 +0200 Subject: [PATCH 17/27] Adding py36 Signed-off-by: onekiloparsec --- .travis.yml | 4 ++++ tox.ini | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index dda11ae..f559426 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,10 @@ env: - TOXENV=py35-django-19 - TOXENV=py35-django-110 - TOXENV=py35-django-111 + - TOXENV=py36-django-18 + - TOXENV=py36-django-19 + - TOXENV=py36-django-110 + - TOXENV=py36-django-111 after_success: coveralls deploy: provider: pypi diff --git a/tox.ini b/tox.ini index d6c80c2..bfb4245 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ downloadcache = {distshare} args_are_paths = false envlist = - {py27,py34,py35}-django-{19,110,111} + {py27,py34,py35,py36}-django-{19,110,111} {py27,py33,py34,py35}-django-18 {py27,py33,py34}-django-17 @@ -12,6 +12,7 @@ basepython = py33: python3.3 py34: python3.4 py35: python3.5 + py36: python3.6 usedevelop = true setenv = CELERY_CONFIG_MODULE=tests.celeryconfig From 3cae97953536091a1f459b87b65456bc12122362 Mon Sep 17 00:00:00 2001 From: onekiloparsec Date: Wed, 19 Jul 2017 16:46:34 +0200 Subject: [PATCH 18/27] Adding missing methods introduced with Django 1.10 Signed-off-by: onekiloparsec --- queued_storage/backends.py | 65 ++++++++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/queued_storage/backends.py b/queued_storage/backends.py index 445b49d..1aa7ee6 100644 --- a/queued_storage/backends.py +++ b/queued_storage/backends.py @@ -16,7 +16,6 @@ class LazyBackend(SimpleLazyObject): - def __init__(self, import_path, options): backend = import_attribute(import_path) super(LazyBackend, self).__init__(lambda: backend(**options)) @@ -86,6 +85,8 @@ def __init__(self, local=None, remote=None, self.remote = self._load_backend(backend=self.remote_path, options=self.remote_options) + # Using the mis-named _load_backend method to get the task instance from the task name string (both self.task...), + # using the tricky method import_attribute. Hard to read. Hard to test... self.task = self._load_backend(backend=task or self.task, handler=import_attribute) if delayed is not None: @@ -96,8 +97,7 @@ def __init__(self, local=None, remote=None, def _load_backend(self, backend=None, options=None, handler=LazyBackend): if backend is None: # pragma: no cover raise ImproperlyConfigured("The QueuedStorage class '%s' " - "doesn't define a needed backend." % - (self)) + "doesn't define a needed backend." % (self)) if not isinstance(backend, six.string_types): raise ImproperlyConfigured("The QueuedStorage class '%s' " "requires its backends to be " @@ -326,7 +326,21 @@ def accessed_time(self, name): :type name: str :rtype: :class:`~python:datetime.datetime` """ - return self.get_storage(name).accessed_time(name) + if VERSION[1] >= 10: + return self.get_storage(name).get_accessed_time(name) + else: + return self.get_storage(name).accessed_time(name) + + def get_accessed_time(self, name): + """ + Returns the last accessed time (as datetime object) of the file + specified by name. + + :param name: file name + :type name: str + :rtype: :class:`~python:datetime.datetime` + """ + return self.accessed_time(name) def created_time(self, name): """ @@ -337,7 +351,21 @@ def created_time(self, name): :type name: str :rtype: :class:`~python:datetime.datetime` """ - return self.get_storage(name).created_time(name) + if VERSION[1] >= 10: + return self.get_storage(name).get_created_time(name) + else: + return self.get_storage(name).created_time(name) + + def get_created_time(self, name): + """ + Returns the creation time (as datetime object) of the file + specified by name. + + :param name: file name + :type name: str + :rtype: :class:`~python:datetime.datetime` + """ + return self.created_time(name) def modified_time(self, name): """ @@ -348,7 +376,23 @@ def modified_time(self, name): :type name: str :rtype: :class:`~python:datetime.datetime` """ - return self.get_storage(name).modified_time(name) + if VERSION[1] >= 10: + return self.get_storage(name).get_modified_time(name) + else: + return self.get_storage(name).modified_time(name) + + def get_modified_time(self, name): + """ + Returns the last modified time (as datetime object) of the file + specified by name. + + :param name: file name + :type name: str + :rtype: :class:`~python:datetime.datetime` + """ + return self.modified_time(name) + + if VERSION[1] >= 7: QueuedStorage = deconstructible(QueuedStorage) @@ -360,6 +404,7 @@ class QueuedFileSystemStorage(QueuedStorage): :class:`~django:django.core.files.storage.FileSystemStorage` as the local storage. """ + def __init__(self, local='django.core.files.storage.FileSystemStorage', *args, **kwargs): super(QueuedFileSystemStorage, self).__init__(local=local, *args, **kwargs) @@ -371,6 +416,7 @@ class QueuedS3BotoStorage(QueuedFileSystemStorage): `django-storages `_ app as the remote storage. """ + def __init__(self, remote='storages.backends.s3boto.S3BotoStorage', *args, **kwargs): super(QueuedS3BotoStorage, self).__init__(remote=remote, *args, **kwargs) @@ -382,6 +428,7 @@ class QueuedCouchDBStorage(QueuedFileSystemStorage): `django-storages `_ app as the remote storage. """ + def __init__(self, remote='storages.backends.couchdb.CouchDBStorage', *args, **kwargs): super(QueuedCouchDBStorage, self).__init__(remote=remote, *args, **kwargs) @@ -393,6 +440,7 @@ class QueuedDatabaseStorage(QueuedFileSystemStorage): `django-storages `_ app as the remote storage. """ + def __init__(self, remote='storages.backends.database.DatabaseStorage', *args, **kwargs): super(QueuedDatabaseStorage, self).__init__(remote=remote, *args, **kwargs) @@ -404,6 +452,7 @@ class QueuedFTPStorage(QueuedFileSystemStorage): `django-storages `_ app as the remote storage. """ + def __init__(self, remote='storages.backends.ftp.FTPStorage', *args, **kwargs): super(QueuedFTPStorage, self).__init__(remote=remote, *args, **kwargs) @@ -415,6 +464,7 @@ class QueuedMogileFSStorage(QueuedFileSystemStorage): `django-storages `_ app as the remote storage. """ + def __init__(self, remote='storages.backends.mogile.MogileFSStorage', *args, **kwargs): super(QueuedMogileFSStorage, self).__init__(remote=remote, *args, **kwargs) @@ -426,6 +476,7 @@ class QueuedGridFSStorage(QueuedFileSystemStorage): `django-storages `_ app as the remote storage. """ + def __init__(self, remote='storages.backends.mongodb.GridFSStorage', *args, **kwargs): super(QueuedGridFSStorage, self).__init__(remote=remote, *args, **kwargs) @@ -437,6 +488,7 @@ class QueuedCloudFilesStorage(QueuedFileSystemStorage): `django-storages `_ app as the remote storage. """ + def __init__(self, remote='storages.backends.mosso.CloudFilesStorage', *args, **kwargs): super(QueuedCloudFilesStorage, self).__init__(remote=remote, *args, **kwargs) @@ -448,5 +500,6 @@ class QueuedSFTPStorage(QueuedFileSystemStorage): `django-storages `_ app as the remote storage. """ + def __init__(self, remote='storages.backends.sftpstorage.SFTPStorage', *args, **kwargs): super(QueuedSFTPStorage, self).__init__(remote=remote, *args, **kwargs) From b9ddf8b9be62492f8e5221c42b99620e639bfde0 Mon Sep 17 00:00:00 2001 From: onekiloparsec Date: Wed, 19 Jul 2017 16:46:50 +0200 Subject: [PATCH 19/27] Small convenience methods Signed-off-by: onekiloparsec --- queued_storage/fields.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/queued_storage/fields.py b/queued_storage/fields.py index 97d62aa..7d14554 100644 --- a/queued_storage/fields.py +++ b/queued_storage/fields.py @@ -7,12 +7,19 @@ class QueuedFieldFile(FieldFile): additional method to transfer the file to the remote storage using the backend's ``transfer`` method. """ + def transfer(self): """ Transfers the file using the storage backend. """ return self.storage.transfer(self.name) + def is_stored_locally(self): + return self.storage.using_local(self.name) + + def is_stored_remotely(self): + return self.storage.using_remote(self.name) + class QueuedFileField(FileField): """ From f7607d3828d5ffd1afd455e1b121f9396902151c Mon Sep 17 00:00:00 2001 From: onekiloparsec Date: Wed, 19 Jul 2017 16:47:02 +0200 Subject: [PATCH 20/27] Specifying python 3.5 and 3.6 in setup Signed-off-by: onekiloparsec --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index e7db38e..1c69190 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,8 @@ def read(*parts): 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Utilities', ], install_requires=[ From 93123f8b948d9c01c47d172d3af108373b47bb63 Mon Sep 17 00:00:00 2001 From: onekiloparsec Date: Wed, 19 Jul 2017 21:44:11 +0200 Subject: [PATCH 21/27] Changed Tasks parameters to avoid re-creation of new Storage class instances. First it makes no sense. But new instances have different directories in case of FileSystemStorage, and this makes testing fail. Signed-off-by: onekiloparsec --- queued_storage/backends.py | 4 +--- queued_storage/tasks.py | 15 +++++---------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/queued_storage/backends.py b/queued_storage/backends.py index 1aa7ee6..cf23515 100644 --- a/queued_storage/backends.py +++ b/queued_storage/backends.py @@ -212,9 +212,7 @@ def transfer(self, name, cache_key=None): """ if cache_key is None: cache_key = self.get_cache_key(name) - return self.task.delay(name, cache_key, - self.local_path, self.remote_path, - self.local_options, self.remote_options) + return self.task.delay(name, cache_key, self.local, self.remote) def get_valid_name(self, name): """ diff --git a/queued_storage/tasks.py b/queued_storage/tasks.py index a2c8ace..80cf428 100644 --- a/queued_storage/tasks.py +++ b/queued_storage/tasks.py @@ -1,12 +1,12 @@ from django.core.cache import cache from celery.task import Task + try: from celery.utils.log import get_task_logger except ImportError: from celery.log import get_task_logger - from .conf import settings from .signals import file_transferred from .utils import import_attribute @@ -60,9 +60,7 @@ def transfer(self, *args, **kwargs): #: :attr:`~queued_storage.conf.settings.QUEUED_STORAGE_RETRY_DELAY`) default_retry_delay = settings.QUEUED_STORAGE_RETRY_DELAY - def run(self, name, cache_key, - local_path, remote_path, - local_options, remote_options, **kwargs): + def run(self, name, cache_key, local, remote, **kwargs): """ The main work horse of the transfer task. Calls the transfer method with the local and remote storage backends as given @@ -82,8 +80,6 @@ def run(self, name, cache_key, :type cache_key: str :rtype: task result """ - local = import_attribute(local_path)(**local_options) - remote = import_attribute(remote_path)(**remote_options) result = self.transfer(name, local, remote, **kwargs) if result is True: @@ -91,8 +87,7 @@ def run(self, name, cache_key, file_transferred.send(sender=self.__class__, name=name, local=local, remote=remote) elif result is False: - args = [name, cache_key, local_path, - remote_path, local_options, remote_options] + args = [name, cache_key, local, remote] self.retry(args=args, kwargs=kwargs) else: raise ValueError("Task '%s' did not return True/False but %s" % @@ -127,9 +122,9 @@ class TransferAndDelete(Transfer): file with the given name using the local storage if the transfer was successful. """ + def transfer(self, name, local, remote, **kwargs): - result = super(TransferAndDelete, self).transfer(name, local, - remote, **kwargs) + result = super(TransferAndDelete, self).transfer(name, local, remote, **kwargs) if result: local.delete(name) return result From f1158513514f1e76fe1cd6f4516453e93d81f7f0 Mon Sep 17 00:00:00 2001 From: onekiloparsec Date: Wed, 19 Jul 2017 21:44:43 +0200 Subject: [PATCH 22/27] Small refactor + allow to inject field storage dynamically. Signed-off-by: onekiloparsec --- tests/models.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/models.py b/tests/models.py index ab6855c..68e647b 100644 --- a/tests/models.py +++ b/tests/models.py @@ -4,7 +4,11 @@ class TestModel(models.Model): - testfile = models.FileField(upload_to='test', null=True) - remote = QueuedFileField(upload_to='test', null=True) - + normal_file = models.FileField(upload_to='test_normal', null=True) + queued_file = QueuedFileField(upload_to='test_queued', null=True) retried = False + + @staticmethod + def queued_file_field_use_storage(storage): + "Small static method to inject storage of the queued_file field dynamically." + TestModel._meta.get_field('queued_file').storage = storage From d08dec776e8a716b47b7f9233b8315190cca8418 Mon Sep 17 00:00:00 2001 From: onekiloparsec Date: Wed, 19 Jul 2017 21:45:08 +0200 Subject: [PATCH 23/27] Convenience backend subclasses for easier debugging. Signed-off-by: onekiloparsec --- tests/backends.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 tests/backends.py diff --git a/tests/backends.py b/tests/backends.py new file mode 100644 index 0000000..8ed8a6c --- /dev/null +++ b/tests/backends.py @@ -0,0 +1,23 @@ +import sys +import tempfile +from django.core.files.storage import FileSystemStorage, Storage + + +# Helper classes for easing the debugging + +class LocalStorage(FileSystemStorage): + def __init__(self, location=None, base_url=None, file_permissions_mode=None, directory_permissions_mode=None): + if location is None: + location = tempfile.mkdtemp() + sys.stdout.write('---> local ' + location + '\n') + super(LocalStorage, self).__init__(location=location, base_url=base_url, file_permissions_mode=file_permissions_mode, + directory_permissions_mode=directory_permissions_mode) + + +class RemoteStorage(FileSystemStorage): + def __init__(self, location=None, base_url=None, file_permissions_mode=None, directory_permissions_mode=None): + if location is None: + location = tempfile.mkdtemp() + sys.stdout.write('---> remote ' + location + '\n') + super(RemoteStorage, self).__init__(location=location, base_url=base_url, file_permissions_mode=file_permissions_mode, + directory_permissions_mode=directory_permissions_mode) From d30cd70d5050fbacd3bc6276cbe116d03ae8f3e5 Mon Sep 17 00:00:00 2001 From: onekiloparsec Date: Thu, 20 Jul 2017 08:32:29 +0200 Subject: [PATCH 24/27] No need to (badly) mock transfer task + small refactor Signed-off-by: onekiloparsec --- tests/tasks.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/tests/tasks.py b/tests/tasks.py index e5c0882..949dd2c 100644 --- a/tests/tasks.py +++ b/tests/tasks.py @@ -1,29 +1,14 @@ from queued_storage.tasks import Transfer -from queued_storage.utils import import_attribute from .models import TestModel -def test_task(name, cache_key, - local_path, remote_path, - local_options, remote_options): - local = import_attribute(local_path)(**local_options) - remote = import_attribute(remote_path)(**remote_options) - remote.save(name, local.open(name)) - - -def delay(*args, **kwargs): - test_task(*args, **kwargs) - -test_task.delay = delay - - -class NoneReturningTask(Transfer): +class NoneReturningTransferTask(Transfer): def transfer(self, *args, **kwargs): return None -class RetryingTask(Transfer): +class RetryingTransferTask(Transfer): def transfer(self, *args, **kwargs): if TestModel.retried: return True From 981c00e275346204a27b1c034a952c552705f1a8 Mon Sep 17 00:00:00 2001 From: onekiloparsec Date: Thu, 20 Jul 2017 08:34:20 +0200 Subject: [PATCH 25/27] Overhaul of unit tests of queued storage. Works with Django 1.10+ Signed-off-by: onekiloparsec --- tests/test_storages.py | 305 ++++++++++++++++++----------------------- 1 file changed, 131 insertions(+), 174 deletions(-) diff --git a/tests/test_storages.py b/tests/test_storages.py index 1b27ffd..f2759ad 100644 --- a/tests/test_storages.py +++ b/tests/test_storages.py @@ -5,14 +5,12 @@ storage systems, this should work as transparently as using one (or even two!) remote storage systems. """ -import os -import shutil + import tempfile from os import path -from datetime import datetime -from django.core.files.base import File -from django.core.files.storage import FileSystemStorage, Storage +from django.core.files.storage import Storage +from django.core.files.uploadedfile import SimpleUploadedFile from django.conf import settings as django_settings from django.test import TestCase @@ -20,32 +18,20 @@ from queued_storage.conf import settings from . import models +from . import backends class StorageTests(TestCase): - def setUp(self): - self.old_celery_always_eager = getattr( - settings, 'CELERY_ALWAYS_EAGER', False) + self.old_celery_always_eager = getattr(settings, 'CELERY_ALWAYS_EAGER', False) settings.CELERY_ALWAYS_EAGER = True - tmp_dir = tempfile.mkdtemp() - self.test_file_name = 'queued_storage.txt' - self.test_file_path = path.join(tmp_dir, self.test_file_name) - with open(self.test_file_path, 'a') as test_file: - test_file.write('test') - self.test_file = open(self.test_file_path, 'r') - - # Overriding MEDIA_ROOT for tests (probbaly undefined at that stage anyway) - # Starting with Django>=1.10, Storages APIs don't allow to write outside MEDIA_ROOT - django_settings.MEDIA_ROOT = tempfile.mkdtemp() - self.local_dir = os.path.join(django_settings.MEDIA_ROOT, 'storage_tests_local') - self.remote_dir = os.path.join(django_settings.MEDIA_ROOT, 'storage_tests_remote') - os.mkdir(self.local_dir) - os.mkdir(self.remote_dir) - - self.addCleanup(shutil.rmtree, self.local_dir) - self.addCleanup(shutil.rmtree, self.remote_dir) - self.addCleanup(shutil.rmtree, tmp_dir) + + # Note that starting with Django>=1.10, Storages APIs don't allow to write outside MEDIA_ROOT + # django_settings.MEDIA_ROOT = tempfile.mkdtemp() + # django_settings.DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' + + self.test_file_name = 'dummy_test_file_name.txt' + self.test_file = SimpleUploadedFile(self.test_file_name, b'these bytes are the file content!') def tearDown(self): settings.CELERY_ALWAYS_EAGER = self.old_celery_always_eager @@ -55,26 +41,28 @@ def test_storage_init(self): Make sure that creating a QueuedStorage object works """ storage = QueuedStorage( - 'django.core.files.storage.FileSystemStorage', - 'django.core.files.storage.FileSystemStorage') + 'tests.backends.LocalStorage', + 'tests.backends.RemoteStorage') + self.assertIsInstance(storage, QueuedStorage) - self.assertEqual(FileSystemStorage, storage.local.__class__) - self.assertEqual(FileSystemStorage, storage.remote.__class__) + self.assertEqual(backends.LocalStorage, storage.local.__class__) + self.assertEqual(backends.RemoteStorage, storage.remote.__class__) def test_storage_cache_key(self): storage = QueuedStorage( - 'django.core.files.storage.FileSystemStorage', - 'django.core.files.storage.FileSystemStorage', + 'tests.backends.LocalStorage', + 'tests.backends.RemoteStorage', cache_prefix='test_cache_key') + self.assertEqual(storage.cache_prefix, 'test_cache_key') def test_storage_methods(self): """ - Make sure that QueuedStorage implements all the methods + Make sure that QueuedStorage implements all the methods of the base Storage class. """ storage = QueuedStorage( - 'django.core.files.storage.FileSystemStorage', - 'django.core.files.storage.FileSystemStorage') + 'tests.backends.LocalStorage', + 'tests.backends.RemoteStorage') file_storage = Storage() @@ -88,189 +76,158 @@ def test_storage_methods(self): self.assertTrue(callable(method), "QueuedStorage has no method '%s'" % attr) - def test_storage_simple_save(self): + def test_storage_simple_local_save_then_transfer(self): """ - Make sure that saving to remote locations actually works + Make sure that saving to remote location actually works. Be careful, by default the transfer task does NOT include a local delete. """ storage = QueuedStorage( - local='django.core.files.storage.FileSystemStorage', - remote='django.core.files.storage.FileSystemStorage', - local_options=dict(location=self.local_dir), - remote_options=dict(location=self.remote_dir), - task='tests.tasks.test_task') + 'tests.backends.LocalStorage', + 'tests.backends.RemoteStorage', + task='queued_storage.tasks.Transfer', + delayed=True) + + models.TestModel.queued_file_field_use_storage(storage) + + obj = models.TestModel(queued_file=self.test_file) - field = models.TestModel._meta.get_field('testfile') - field.storage = storage + self.assertEqual(storage, obj.queued_file.storage) + self.assertIsNone(getattr(obj.queued_file.storage, 'result', None)) - obj = models.TestModel(testfile=File(self.test_file)) obj.save() - self.assertTrue(path.isfile(path.join(self.local_dir, obj.testfile.name))) - self.assertTrue(path.isfile(path.join(self.remote_dir, obj.testfile.name))) + self.assertTrue(obj.queued_file.is_stored_locally()) + self.assertFalse(obj.queued_file.is_stored_remotely()) - def test_storage_celery_save(self): - """ - Make sure it actually works when using Celery as a task queue - """ - storage = QueuedStorage( - local='django.core.files.storage.FileSystemStorage', - remote='django.core.files.storage.FileSystemStorage', - local_options=dict(location=self.local_dir), - remote_options=dict(location=self.remote_dir)) + # We shouldn't have to test this, as it is purely internal implementation of QueuedStorage. But because, it is + # FileSystemStorage under the hood, we could have a sneak peak... + upload_to_folder_name = models.TestModel._meta.get_field('queued_file').upload_to + sub_folder_file_path = path.join(upload_to_folder_name, self.test_file_name) - field = models.TestModel._meta.get_field('testfile') - field.storage = storage + # Test if file is here. + self.assertTrue(path.isfile(obj.queued_file.storage.local.path(sub_folder_file_path))) + # Test if file is, of course, not there. + self.assertFalse(path.isfile(obj.queued_file.storage.remote.path(sub_folder_file_path))) - obj = models.TestModel(testfile=File(self.test_file)) - obj.save() + # self.assertEqual(storage.listdir('test')[1], [self.test_file_name]) + # self.assertEqual(storage.size(subdir_path), os.stat(self.test_file_path).st_size) + # self.assertEqual(storage.url(self.test_file_name), self.test_file_name) + # self.assertIsInstance(storage.accessed_time(subdir_path), datetime) + # self.assertIsInstance(storage.created_time(subdir_path), datetime) + # self.assertIsInstance(storage.modified_time(subdir_path), datetime) + + # WARNING: By default the transfer task does NOT include a local delete. + obj.queued_file.transfer() - self.assertTrue(obj.testfile.storage.result.get()) - self.assertTrue(path.isfile(path.join(self.local_dir, obj.testfile.name))) - self.assertTrue( - path.isfile(path.join(self.remote_dir, obj.testfile.name)), - "Remote file is not available.") - self.assertFalse(storage.using_local(obj.testfile.name)) - self.assertTrue(storage.using_remote(obj.testfile.name)) - - self.assertEqual(self.test_file_name, - storage.get_valid_name(self.test_file_name)) - self.assertEqual(self.test_file_name, - storage.get_available_name(self.test_file_name)) - - subdir_path = os.path.join('test', self.test_file_name) - self.assertTrue(storage.exists(subdir_path)) - self.assertEqual(storage.path(self.test_file_name), - path.join(self.local_dir, self.test_file_name)) - self.assertEqual(storage.listdir('test')[1], [self.test_file_name]) - self.assertEqual(storage.size(subdir_path), - os.stat(self.test_file_path).st_size) - self.assertEqual(storage.url(self.test_file_name), self.test_file_name) - self.assertIsInstance(storage.accessed_time(subdir_path), datetime) - self.assertIsInstance(storage.created_time(subdir_path), datetime) - self.assertIsInstance(storage.modified_time(subdir_path), datetime) - - subdir_name = 'queued_storage_2.txt' - testfile = storage.open(subdir_name, 'w') - try: - testfile.write('test') - finally: - testfile.close() - self.assertTrue(storage.exists(subdir_name)) - storage.delete(subdir_name) - self.assertFalse(storage.exists(subdir_name)) - - def test_transfer_and_delete(self): + self.assertFalse(obj.queued_file.is_stored_locally()) + self.assertTrue(obj.queued_file.is_stored_remotely()) + + # Test if file is still here. + self.assertTrue(path.isfile(obj.queued_file.storage.local.path(sub_folder_file_path))) + # Test if file is now there too. + self.assertTrue(path.isfile(obj.queued_file.storage.remote.path(sub_folder_file_path))) + + def test_storage_simple_local_save_then_transfer_and_delete(self): """ - Make sure the TransferAndDelete task does what it says + Make sure that saving to remote location actually works. """ storage = QueuedStorage( - local='django.core.files.storage.FileSystemStorage', - remote='django.core.files.storage.FileSystemStorage', - local_options=dict(location=self.local_dir), - remote_options=dict(location=self.remote_dir), - task='queued_storage.tasks.TransferAndDelete') + 'tests.backends.LocalStorage', + 'tests.backends.RemoteStorage', + task='queued_storage.tasks.TransferAndDelete', + delayed=True) - field = models.TestModel._meta.get_field('testfile') - field.storage = storage + models.TestModel.queued_file_field_use_storage(storage) - obj = models.TestModel(testfile=File(self.test_file)) + obj = models.TestModel(queued_file=self.test_file) obj.save() - obj.testfile.storage.result.get() + self.assertTrue(obj.queued_file.is_stored_locally()) + self.assertFalse(obj.queued_file.is_stored_remotely()) - self.assertFalse( - path.isfile(path.join(self.local_dir, obj.testfile.name)), - "Local file is still available") - self.assertTrue( - path.isfile(path.join(self.remote_dir, obj.testfile.name)), - "Remote file is not available.") + obj.queued_file.transfer() - def test_transfer_returns_boolean(self): + self.assertTrue(obj.queued_file.is_stored_remotely()) + self.assertFalse(obj.queued_file.is_stored_locally()) + + def test_storage_celery_save(self): """ - Make sure an exception is thrown when the transfer task does not return - a boolean. We don't want to confuse Celery. + Make sure that saving to remote location actually works. """ storage = QueuedStorage( - local='django.core.files.storage.FileSystemStorage', - remote='django.core.files.storage.FileSystemStorage', - local_options=dict(location=self.local_dir), - remote_options=dict(location=self.remote_dir), - task='tests.tasks.NoneReturningTask') + 'tests.backends.LocalStorage', + 'tests.backends.RemoteStorage', + task='queued_storage.tasks.Transfer', + delayed=False) - field = models.TestModel._meta.get_field('testfile') - field.storage = storage + models.TestModel.queued_file_field_use_storage(storage) - obj = models.TestModel(testfile=File(self.test_file)) + obj = models.TestModel(queued_file=self.test_file) obj.save() - self.assertRaises(ValueError, - obj.testfile.storage.result.get, propagate=True) + self.assertFalse(obj.queued_file.is_stored_locally()) + self.assertTrue(obj.queued_file.is_stored_remotely()) - def test_transfer_retried(self): + # Test that calling transfer at that point has no effects. + obj.queued_file.transfer() + self.assertFalse(obj.queued_file.is_stored_locally()) + self.assertTrue(obj.queued_file.is_stored_remotely()) + + def test_storage_celery_save_with_delete(self): """ - Make sure the transfer task is retried correctly. + Make sure that saving to remote location actually works. """ storage = QueuedStorage( - local='django.core.files.storage.FileSystemStorage', - remote='django.core.files.storage.FileSystemStorage', - local_options=dict(location=self.local_dir), - remote_options=dict(location=self.remote_dir), - task='tests.tasks.RetryingTask') - field = models.TestModel._meta.get_field('testfile') - field.storage = storage + 'tests.backends.LocalStorage', + 'tests.backends.RemoteStorage', + task='queued_storage.tasks.TransferAndDelete', + delayed=False) - self.assertFalse(models.TestModel.retried) + models.TestModel.queued_file_field_use_storage(storage) - obj = models.TestModel(testfile=File(self.test_file)) + obj = models.TestModel(queued_file=self.test_file) obj.save() - self.assertFalse(obj.testfile.storage.result.get()) - self.assertTrue(models.TestModel.retried) + self.assertFalse(obj.queued_file.is_stored_locally()) + self.assertTrue(obj.queued_file.is_stored_remotely()) + + # Test that calling transfer at that point has no effects. + obj.queued_file.transfer() + self.assertFalse(obj.queued_file.is_stored_locally()) + self.assertTrue(obj.queued_file.is_stored_remotely()) - def test_delayed_storage(self): + def test_transfer_returns_boolean(self): + """ + Make sure an exception is thrown when the transfer task does not return + a boolean. We don't want to confuse Celery. + """ storage = QueuedStorage( - local='django.core.files.storage.FileSystemStorage', - remote='django.core.files.storage.FileSystemStorage', - local_options=dict(location=self.local_dir), - remote_options=dict(location=self.remote_dir), - delayed=True) + 'tests.backends.LocalStorage', + 'tests.backends.RemoteStorage', + task='tests.tasks.NoneReturningTransferTask', + delayed=False) - field = models.TestModel._meta.get_field('testfile') - field.storage = storage + models.TestModel.queued_file_field_use_storage(storage) - obj = models.TestModel(testfile=File(self.test_file)) + obj = models.TestModel(queued_file=self.test_file) obj.save() - self.assertIsNone(getattr(obj.testfile.storage, 'result', None)) + self.assertRaises(ValueError, obj.queued_file.storage.result.get, propagate=True) - self.assertFalse( - path.isfile(path.join(self.remote_dir, obj.testfile.name)), - "Remote file should not be transferred automatically.") - - result = obj.testfile.storage.transfer(obj.testfile.name) - result.get() - - self.assertTrue( - path.isfile(path.join(self.remote_dir, obj.testfile.name)), - "Remote file is not available.") - - def test_remote_file_field(self): + def test_transfer_retried(self): + """ + Make sure the transfer task is retried correctly. + """ storage = QueuedStorage( - local='django.core.files.storage.FileSystemStorage', - remote='django.core.files.storage.FileSystemStorage', - local_options=dict(location=self.local_dir), - remote_options=dict(location=self.remote_dir), - delayed=True) + 'tests.backends.LocalStorage', + 'tests.backends.RemoteStorage', + task='tests.tasks.RetryingTransferTask', + delayed=False) - field = models.TestModel._meta.get_field('remote') - field.storage = storage + models.TestModel.queued_file_field_use_storage(storage) + self.assertFalse(models.TestModel.retried) - obj = models.TestModel(remote=File(self.test_file)) + obj = models.TestModel(queued_file=self.test_file) obj.save() - self.assertIsNone(getattr(obj.testfile.storage, 'result', None)) - - result = obj.remote.transfer() - self.assertTrue(result) - self.assertTrue(path.isfile(path.join(self.remote_dir, - obj.remote.name))) + self.assertFalse(obj.queued_file.storage.result.get()) + self.assertTrue(models.TestModel.retried) From b80eb2dafb0010ae38bdd0bbe73d59ff2b7fec48 Mon Sep 17 00:00:00 2001 From: onekiloparsec Date: Thu, 20 Jul 2017 10:14:43 +0200 Subject: [PATCH 26/27] Fixing tox script Signed-off-by: onekiloparsec --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f559426..e27c472 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ cache: directories: - $HOME/.cache/pip install: pip install tox coveralls wheel -script: tox +script: tox -e ${TOXENV} env: - TOXENV=py27-django-17 - TOXENV=py27-django-18 @@ -28,7 +28,6 @@ env: - TOXENV=py35-django-19 - TOXENV=py35-django-110 - TOXENV=py35-django-111 - - TOXENV=py36-django-18 - TOXENV=py36-django-19 - TOXENV=py36-django-110 - TOXENV=py36-django-111 From 89a4b6aaa48c1257f19c5212a8872a182a574746 Mon Sep 17 00:00:00 2001 From: onekiloparsec Date: Thu, 20 Jul 2017 10:25:34 +0200 Subject: [PATCH 27/27] Removed support for python3.6 for now. It makes travis fail with error: py36-django-19 create: /home/travis/build/jazzband/django-queued-storage/.tox/py36-django-19 ERROR: InterpreterNotFound: python3.6 Signed-off-by: onekiloparsec --- .travis.yml | 3 --- setup.py | 1 - tox.ini | 3 +-- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index e27c472..6564dbf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,9 +28,6 @@ env: - TOXENV=py35-django-19 - TOXENV=py35-django-110 - TOXENV=py35-django-111 - - TOXENV=py36-django-19 - - TOXENV=py36-django-110 - - TOXENV=py36-django-111 after_success: coveralls deploy: provider: pypi diff --git a/setup.py b/setup.py index 1c69190..71aaf53 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,6 @@ def read(*parts): 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', 'Topic :: Utilities', ], install_requires=[ diff --git a/tox.ini b/tox.ini index bfb4245..d6c80c2 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ downloadcache = {distshare} args_are_paths = false envlist = - {py27,py34,py35,py36}-django-{19,110,111} + {py27,py34,py35}-django-{19,110,111} {py27,py33,py34,py35}-django-18 {py27,py33,py34}-django-17 @@ -12,7 +12,6 @@ basepython = py33: python3.3 py34: python3.4 py35: python3.5 - py36: python3.6 usedevelop = true setenv = CELERY_CONFIG_MODULE=tests.celeryconfig