REST API (WIP)

requirements/base.txt

@@ -14,9 +14,11 @@ django-extensions==1.7.9
 django-model-utils==3.0.0
 #tastypie 0.13.3 has breaking changes
 django-tastypie==0.13.1
+formencode==1.3.1
 futures==3.0.5  # used by gunicorn's async workers
 gevent==1.2.1  # used by gunicorn's async workers
 gunicorn==19.7.1
+inflect==0.2.1
 jsonfield==2.0.1
 logutils==0.3.4.1
 lxml==3.7.3

requirements/test.txt

@@ -1,6 +1,7 @@
 # Test dependencies go here.
 -r base.txt
 
+django_mock_queries==1.0.5
 pytest
 pytest-cov==2.4.0
 coverage==4.2

storage_service/locations/api/urls.py

@@ -1,6 +1,7 @@
 from django.conf.urls import include, url
 from tastypie.api import Api
 from locations.api import v1, v2
+import locations.api.v3 as v3_api
 
 from locations.api.sword import views
 
@@ -19,6 +20,7 @@
 v2_api.register(v2.AsyncResource())
 
 urlpatterns = [
+    url(r'v3/', include(v3_api.urls)),
     url(r'', include(v1_api.urls)),
     url(r'v1/sword/$', views.service_document, name='sword_service_document'),
     url(r'', include(v2_api.urls)),

storage_service/locations/api/v3/__init__.py

@@ -0,0 +1,68 @@
+"""Version 3 of the Storage Service API.
+
+The Storage Service exposes the following resources via a consistent HTTP JSON
+interface under the path namespace /api/v3/:
+
+- /locations/ --- purpose-specific paths within a /spaces/ resource
+- /packages/ --- Information Package (SIP, DIP or AIP)
+- /spaces/ --- storage space with behaviour specific to backing system
+- /pipelines/ --- an Archivematica instance that is the source of a package
+
+The following resources may be exposed later:
+
+- /file/ --- a file on disk (which is in a package), represented as db row.
+- /fsobjects/ --- directories and files on disk, read-only, no database models
+
+All resources have endpoints that follow this pattern::
+
+    +-----------------+-------------+----------------------------+--------+
+    | Purpose         | HTTP Method | Path                       | Method |
+    +-----------------+-------------+----------------------------+--------+
+    | Create new      | POST        | /<cllctn_name>/            | create |
+    | Create data     | GET         | /<cllctn_name>/new/        | new    |
+    | Read all        | GET         | /<cllctn_name>/            | index  |
+    | Read specific   | GET         | /<cllctn_name>/<id>/       | show   |
+    | Update specific | PUT         | /<cllctn_name>/<id>/       | update |
+    | Update data     | GET         | /<cllctn_name>/<id>/edit/  | edit   |
+    | Delete specific | DELETE      | /<cllctn_name>/<id>/       | delete |
+    | Search          | SEARCH      | /<cllctn_name>/            | search |
+    | Search          | POST        | /<cllctn_name>/search/     | search |
+    | Search data     | GET         | /<cllctn_name>/new_search/ | search |
+    +-----------------+-------------+----------------------------+--------+
+
+.. note:: To remove the search-related routes for a given resource, create a
+   ``'searchable'`` key with value ``False`` in the configuration for the
+   resource in the ``RESOURCES`` dict. E.g., ``'location': {'searchable':
+   False}`` will make the /locations/ resource non-searchable.
+
+.. note:: All resources expose the same endpoints. If a resource needs special
+   treatment, it should be done at the corresponding class level. E.g., if
+   ``POST /packages/`` (creating a package) is special, then do special stuff
+   in ``resources.py::Packages.create``. Similarly, if packages are indelible,
+   then ``resources.py::Packages.delete`` should return 404.
+
+"""
+
+from locations.api.v3.remple import API
+from locations.api.v3.resources import (
+    Locations,
+    Packages,
+    Spaces,
+    Pipelines,
+)
+
+API_VERSION = '3.0.0'
+SERVICE_NAME = 'Archivematica Storage Service'
+
+resources = {
+    'location': {'resource_cls': Locations},
+    'package': {'resource_cls': Packages},
+    'space': {'resource_cls': Spaces},
+    'pipeline': {'resource_cls': Pipelines},
+}
+
+api = API(api_version=API_VERSION, service_name=SERVICE_NAME)
+api.register_resources(resources)
+urls = api.get_urlpatterns()
+
+__all__ = ('urls', 'api')

storage_service/locations/api/v3/remple/README.rst

@@ -0,0 +1,230 @@
+================================================================================
+  Remple: REST Simple
+================================================================================
+
+
+OpenAPI Generate Django Command
+================================================================================
+
+Generate the OpenAPI YAML file for V3 of the Storage Service REST API::
+
+    $ docker-compose exec archivematica-storage-service /src/storage_service/manage.py apiv3openapi
+
+
+Documentation of the API
+================================================================================
+
+Note the following should be superseded by the OpenAPI (Swagger) auto-generated
+interface/documentation. I am keeping it for now.
+
+
+Get all resources of a given type
+--------------------------------------------------------------------------------
+
+Example: GET /pipelines/::
+
+    $ curl -H "Authorization: ApiKey test:test" \
+          http://127.0.0.1:62081/api/v3/pipelines/
+    [
+        {
+            "api_key": "test",
+            "uuid": "3bf15d1c-4c7e-4002-b7b0-668983869d49",
+            "resource_uri": "/api/v3/pipelines/3bf15d1c-4c7e-4002-b7b0-668983869d49/",
+            "enabled": true,
+            "api_username": "test",
+            "remote_name": "172.20.0.13",
+            "id": 1,
+            "description": "Archivematica on f5c59e3ed603"
+        }
+    ]
+
+Pagination works by passing query parameters ``page`` and ``items_per_page``.
+Example: GET /locations/ with pagination::
+
+    $ curl -H "Authorization: ApiKey test:test" \
+          http://127.0.0.1:62081/api/v3/locations/?page=2&items_per_page=2
+    {
+        "paginator": {
+            "count": 7,
+            "items_per_page": 2,
+            "page": 2
+        },
+        "items": [
+            {
+                "pipeline": ["3bf15d1c-4c7e-4002-b7b0-668983869d49"],
+                "used": 0,
+                "uuid": "5dfd0998-35a6-4724-b428-e538a8f2cdd5",
+                "space": "c7463e9b-88d2-4674-a85b-5fc6905fd233",
+                "description": "",
+                "enabled": true,
+                "quota": null,
+                "relative_path": "home",
+                "purpose": "TS",
+                "replicators": [],
+                "id": 1,
+                "resource_uri": "/api/v3/locations/5dfd0998-35a6-4724-b428-e538a8f2cdd5/"
+            },
+            {
+                "pipeline": ["3bf15d1c-4c7e-4002-b7b0-668983869d49"],
+                "used": 0,
+                "uuid": "7b1784b1-8887-453e-9be3-087ab2e0bb63",
+                "space": "c7463e9b-88d2-4674-a85b-5fc6905fd233",
+                "description": "Default transfer backlog",
+                "enabled": true,
+                "quota": null,
+                "relative_path": "var/archivematica/sharedDirectory/www/AIPsStore/transferBacklog",
+                "purpose": "BL",
+                "replicators": [],
+                "id": 4,
+                "resource_uri": "/api/v3/locations/7b1784b1-8887-453e-9be3-087ab2e0bb63/"
+            }
+        ]
+    }
+
+
+Get a single resource by its UUID
+--------------------------------------------------------------------------------
+
+Example: GET /pipelines/<UUID>/::
+
+    $ curl -H "Authorization: ApiKey test:test" \
+          http://127.0.0.1:62081/api/v3/pipelines/3bf15d1c-4c7e-4002-b7b0-668983869d49/
+    {
+        "api_key": "test",
+        "uuid": "3bf15d1c-4c7e-4002-b7b0-668983869d49",
+        "resource_uri": "/api/v3/pipelines/3bf15d1c-4c7e-4002-b7b0-668983869d49/",
+        "enabled": true,
+        "api_username": "test",
+        "remote_name": "172.20.0.13",
+        "id": 1,
+        "description": "Archivematica on f5c59e3ed603"
+    }
+
+
+Search across resources
+--------------------------------------------------------------------------------
+
+Search works by making a ``SEARCH`` request to the standard collection URI of
+the resource (``/resources/``) or a ``POST`` request to ``/resources/search/``,
+e.g., ``/locations/search/``.
+
+The request body should contain a object (dict) that has a ``query`` key and an
+optional ``paginator`` key. The values of both of these keys are objects. The
+``query`` dict has a ``filter`` key and an optional ``order_by`` key. Example::
+
+    {
+        "query": {
+            "filter": ["Location", "purpose", "regex", "[AT]S"]
+            "order_by": [ ... ]
+        },
+        "paginator": { ... }
+    }
+
+Regex search for transfer source and archival storage locations. SEARCH
+/locations/::
+
+    $ curl -H "Authorization: ApiKey test:test" \
+           -H "Content-Type: application/json" \
+           -X SEARCH \
+           -d '{"query": {"filter": ["Location", "purpose", "regex", "[AT]S"]}}' \
+           http://127.0.0.1:62081/api/v3/locations/
+    [
+        {
+            "pipeline": ["3bf15d1c-4c7e-4002-b7b0-668983869d49"],
+            "used": 0,
+            "uuid": "5dfd0998-35a6-4724-b428-e538a8f2cdd5",
+            "space": "c7463e9b-88d2-4674-a85b-5fc6905fd233",
+            "description": "",
+            "enabled": true,
+            "quota": null,
+            "relative_path": "home",
+            "purpose": "TS",
+            "replicators": [],
+            "id": 1,
+            "resource_uri": "/api/v3/locations/5dfd0998-35a6-4724-b428-e538a8f2cdd5/"
+        },
+        {
+            "pipeline": ["3bf15d1c-4c7e-4002-b7b0-668983869d49"],
+            "used": 0,
+            "uuid": "a933c327-f081-4faa-b5dc-a0c81f4f494f",
+            "space": "c7463e9b-88d2-4674-a85b-5fc6905fd233",
+            "description": "Store AIP in standard Archivematica Directory",
+            "enabled": true,
+            "quota": null,
+            "relative_path": "var/archivematica/sharedDirectory/www/AIPsStore",
+            "purpose": "AS",
+            "replicators": [],
+            "id": 2,
+            "resource_uri": "/api/v3/locations/a933c327-f081-4faa-b5dc-a0c81f4f494f/"
+        }
+    ]
+
+The same search as above, but with reverse ordering and using ``POST
+/locations/search/``::
+
+    $ curl -H "Authorization: ApiKey test:test" \
+           -H "Content-Type: application/json" \
+           -X POST \
+           -d '{"query": {"filter": ["Location", "purpose", "regex", "[AT]S"], "order_by": [["purpose"]]}}' \
+           http://127.0.0.1:62081/api/v3/locations/search/
+    [
+        {
+            "pipeline": ["3bf15d1c-4c7e-4002-b7b0-668983869d49" ],
+            "used": 0,
+            "uuid": "a933c327-f081-4faa-b5dc-a0c81f4f494f",
+            "space": "c7463e9b-88d2-4674-a85b-5fc6905fd233",
+            "description": "Store AIP in standard Archivematica Directory",
+            "enabled": true,
+            "quota": null,
+            "relative_path": "var/archivematica/sharedDirectory/www/AIPsStore",
+            "purpose": "AS",
+            "replicators": [],
+            "id": 2,
+            "resource_uri": "/api/v3/locations/a933c327-f081-4faa-b5dc-a0c81f4f494f/"
+        },
+        {
+            "pipeline": ["3bf15d1c-4c7e-4002-b7b0-668983869d49"],
+            "used": 0,
+            "uuid": "5dfd0998-35a6-4724-b428-e538a8f2cdd5",
+            "space": "c7463e9b-88d2-4674-a85b-5fc6905fd233",
+            "description": "",
+            "enabled": true,
+            "quota": null,
+            "relative_path": "home",
+            "purpose": "TS",
+            "replicators": [],
+            "id": 1,
+            "resource_uri": "/api/v3/locations/5dfd0998-35a6-4724-b428-e538a8f2cdd5/"
+        }
+    ]
+
+The same search as above, this time adding pagination::
+
+    $ curl -H "Authorization: ApiKey test:test" \
+           -H "Content-Type: application/json" \
+           -X POST \
+           -d '{"paginator": {"page": 2, "items_per_page": 1}, "query": {"filter": ["Location", "purpose", "regex", "[AT]S"], "order_by": [["purpose"]]}}' \
+           http://127.0.0.1:62081/api/v3/locations/search/
+    {
+        "paginator": {
+            "count": 2,
+            "items_per_page": 1,
+            "page": 2
+        },
+        "items": [
+            {
+                "pipeline": ["3bf15d1c-4c7e-4002-b7b0-668983869d49"],
+                "used": 0,
+                "uuid": "5dfd0998-35a6-4724-b428-e538a8f2cdd5",
+                "space": "c7463e9b-88d2-4674-a85b-5fc6905fd233",
+                "description": "",
+                "enabled": true,
+                "quota": null,
+                "relative_path": "home",
+                "purpose": "TS",
+                "replicators": [],
+                "id": 1,
+                "resource_uri": "/api/v3/locations/5dfd0998-35a6-4724-b428-e538a8f2cdd5/"
+            }
+        ]
+    }

storage_service/locations/api/v3/remple/__init__.py

@@ -0,0 +1,21 @@
+"""Remple: REST Simple
+
+Usage::
+
+    >>> from remple import remple, Resources
+    >>> class Users(remple.Resources):
+    ...     model_cls = User  # A Django model class
+    ...     schema_cls = UserSchema  # A Formencode class
+    >>> resources = {'user': {'resource_cls': Users}}
+    >>> remple.register_resources(resources)
+    >>> urls = remple.get_urlpatterns()  # Include thes in Django urlpatterns
+"""
+
+from __future__ import absolute_import
+
+from .resources import Resources, ReadonlyResources
+from .routebuilder import RouteBuilder as API
+from . import utils
+from .schemata import ValidModelObject
+
+__all__ = ('API', 'utils', 'Resources', 'ReadonlyResources', 'ValidModelObject')