File storage interface improvements

LICENSE

@@ -1,6 +1,7 @@
 MIT License
 
-Copyright (C) 2015-2019 CERN.
+Copyright (C) 2015-2020 CERN.
+Copyright (C) 2020 Cottage Labs LLP.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

docs/index.rst

@@ -22,6 +22,7 @@ Invenio-Files-REST.
    installation
    configuration
    usage
+   new-storage-backend
 
 
 API Reference

docs/new-storage-backend.rst

@@ -0,0 +1,98 @@
+..
+    This file is part of Invenio.
+    Copyright (C) 2020 Cottage Labs LLP
+
+    Invenio is free software; you can redistribute it and/or modify it
+    under the terms of the MIT License; see LICENSE file for more details.
+
+
+Developing a new storage backend
+================================
+
+A storage backend should subclass ``invenio_files_rest.storage.StorageBackend`` and should minimally implement the
+``open()``, ``delete()``, ``_initialize(size)``, ``get_save_stream()`` and ``update_stream(seek)`` methods. You should
+register the backend with an entry point in your ``setup.py``:
+
+.. code-block:: python
+
+   setup(
+       ...,
+       entry_points={
+           ...,
+           'invenio_files_rest.storage': [
+               'mybackend = mypackage.storage:MyStorageBackend',
+           ],
+           ...
+       },
+       ...
+   )
+
+Implementation
+--------------
+
+The base class handles reporting progress, file size limits and checksumming.
+
+Here's an example implementation of a storage backend that stores files remotely over HTTP with no authentication.
+
+.. code-block:: python
+
+   import contextlib
+   import httplib
+   import urllib
+   import urllib.parse
+
+   from invenio_files_rest.storage import StorageBackend
+
+
+   class StorageBackend(StorageBackend):
+       def open(self):
+           return contextlib.closing(
+               urllib.urlopen('GET', self.uri)
+           )
+
+       def _initialize(self, size=0):
+           # Allocate space for the file. You can use `self.uri` as a suggested location, or return
+           # a new location as e.g. `{"uri": the_new_uri}`.
+           urllib.urlopen('POST', self.uri, headers={'X-Expected-Size': str(size)})
+           return {}
+
+       @contextlib.contextmanager
+       def get_save_stream(self):
+           # This should be a context manager (i.e. something that can be used in a `with` statement)
+           # which closes the file when exiting the context manager and performs any clear-up if
+           # an error occurs.
+           parsed_uri = urllib.parse.urlparse(self.uri)
+           # Assume the URI is HTTP, not HTTPS, and doesn't have a port or querystring
+           conn = httplib.HTTPConnection(parsed_uri.netloc)
+
+           conn.putrequest('PUT', parsed_uri.path)
+           conn.endheaders()
+
+           # HTTPConnections have a `send` method, whereas a file-like object should have `write`
+           conn.write = conn.send
+
+           try:
+               yield conn.send
+               response = conn.getresponse()
+               if not 200 <= response.status < 300:
+                   raise IOError("HTTP error")
+           finally:
+               conn.close()
+
+
+Checksumming
+------------
+
+The base class performs checksumming by default, using the ``checksum_hash_name`` class or instance attribute as
+the hashlib hashing function to use. If your underlying storage system provides checksumming functionality you can set
+this to ``None`` and override ``checksum()``:
+
+.. code-block:: python
+
+   class RemoteChecksumStorageBackend(StorageBackend):
+       checksum_hash_name = None
+
+       def checksum(self, chunk_size=None):
+           checksum = urllib.urlopen(self.uri + '?checksum=sha256').read()
+           return f'sha256:{checksum}'
+

invenio_files_rest/alembic/0999e27defd5_add_backend_name.py

@@ -0,0 +1,41 @@
+#
+# This file is part of Invenio.
+# Copyright (C) 2020 Cottage Labs LLP
+#
+# Invenio is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Add backend_name."""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '0999e27defd5'
+down_revision = '8ae99b034410'
+branch_labels = ()
+depends_on = None
+
+
+def upgrade():
+    """Upgrade database."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        'files_files',
+        sa.Column('storage_backend',
+                  sa.String(length=32),
+                  nullable=True))
+    op.add_column(
+        'files_location',
+        sa.Column('storage_backend',
+                  sa.String(length=32),
+                  nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    """Downgrade database."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('files_location', 'storage_backend')
+    op.drop_column('files_files', 'storage_backend')
+    # ### end Alembic commands ###

invenio_files_rest/config.py

@@ -1,13 +1,15 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of Invenio.
-# Copyright (C) 2015-2019 CERN.
+# Copyright (C) 2015-2020 CERN.
+# Copyright (C) 2020 Cottage Labs LLP
 #
 # Invenio is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
 
 """Invenio Files Rest module configuration file."""
 
+import pkg_resources
 from datetime import timedelta
 
 from invenio_files_rest.helpers import create_file_streaming_redirect_response
@@ -57,6 +59,15 @@
 FILES_REST_STORAGE_FACTORY = 'invenio_files_rest.storage.pyfs_storage_factory'
 """Import path of factory used to create a storage instance."""
 
+FILES_REST_DEFAULT_STORAGE_BACKEND = 'pyfs'
+"""The default storage backend name."""
+
+FILES_REST_STORAGE_BACKENDS = {
+    ep.name: ep.load()
+    for ep in pkg_resources.iter_entry_points('invenio_files_rest.storage')
+}
+"""A mapping from storage backend names to classes."""
+
 FILES_REST_PERMISSION_FACTORY = \
     'invenio_files_rest.permissions.permission_factory'
 """Permission factory to control the files access from the REST interface."""

invenio_files_rest/ext.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of Invenio.
-# Copyright (C) 2015-2019 CERN.
+# Copyright (C) 2015-2020 CERN.
+# Copyright (C) 2020 Cottage Labs LLP.
 #
 # Invenio is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
@@ -10,6 +11,7 @@
 
 from __future__ import absolute_import, print_function
 
+import warnings
 from flask import abort
 from werkzeug.exceptions import UnprocessableEntity
 from werkzeug.utils import cached_property
@@ -30,9 +32,20 @@ def __init__(self, app):
     @cached_property
     def storage_factory(self):
         """Load default storage factory."""
-        return load_or_import_from_config(
+        if self.app.config['FILES_REST_STORAGE_FACTORY'] in [
+            'invenio_files_rest.storage.pyfs_storage_factory',
+        ]:
+            warnings.warn(DeprecationWarning(
+                "The " + self.app.config['FILES_REST_STORAGE_FACTORY'] +
+                " storage factory has been deprecated in favour of"
+                " 'invenio_files_rest.storage:StorageFactory"
+            ))
+        storage_factory = load_or_import_from_config(
             'FILES_REST_STORAGE_FACTORY', app=self.app
         )
+        if isinstance(storage_factory, type):
+            storage_factory = storage_factory(self.app)
+        return storage_factory
 
     @cached_property
     def permission_factory(self):