diff --git a/designsafe/apps/api/filemeta/__init__.py b/designsafe/apps/api/filemeta/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/designsafe/apps/api/filemeta/apps.py b/designsafe/apps/api/filemeta/apps.py new file mode 100644 index 0000000000..9fd0be8ae0 --- /dev/null +++ b/designsafe/apps/api/filemeta/apps.py @@ -0,0 +1,10 @@ +"""File Meta app config""" +from django.apps import AppConfig + + +class FileMetaAppConfig(AppConfig): + """App config for File Meta""" + + name = "designsafe.apps.api.filemeta" + label = "filemeta_api" + verbose_name = "Designsafe File Meta" diff --git a/designsafe/apps/api/filemeta/migrations/0001_initial.py b/designsafe/apps/api/filemeta/migrations/0001_initial.py new file mode 100644 index 0000000000..4b35130b5d --- /dev/null +++ b/designsafe/apps/api/filemeta/migrations/0001_initial.py @@ -0,0 +1,36 @@ +# Generated by Django 4.2.6 on 2024-04-18 16:44 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="FileMetaModel", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("value", models.JSONField()), + ("created", models.DateTimeField(default=django.utils.timezone.now)), + ("last_updated", models.DateTimeField(auto_now=True)), + ], + options={ + "indexes": [ + models.Index(models.F("value__system"), name="idx_value_system"), + models.Index(models.F("value__path"), name="idx_value_path"), + ], + }, + ), + ] diff --git a/designsafe/apps/api/filemeta/migrations/0002_auto_20240418_1650.py b/designsafe/apps/api/filemeta/migrations/0002_auto_20240418_1650.py new file mode 100644 index 0000000000..382f3f0b93 --- /dev/null +++ b/designsafe/apps/api/filemeta/migrations/0002_auto_20240418_1650.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.6 on 2024-04-18 16:50 +# Custom migration to handle unique constraint of system/path in "value" +from django.db import migrations, models +import django.db.models.expressions as expressions + + +class Migration(migrations.Migration): + dependencies = [ + ("filemeta_api", "0001_initial"), + ] + + operations = [ + migrations.AddConstraint( + model_name="filemetamodel", + constraint=models.UniqueConstraint( + expressions.F("value__system"), + expressions.F("value__path"), + name="unique_system_path", + ), + ), + ] diff --git a/designsafe/apps/api/filemeta/migrations/__init__.py b/designsafe/apps/api/filemeta/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/designsafe/apps/api/filemeta/models.py b/designsafe/apps/api/filemeta/models.py new file mode 100644 index 0000000000..e9a9e41e0e --- /dev/null +++ b/designsafe/apps/api/filemeta/models.py @@ -0,0 +1,67 @@ +"""File Meta model""" + +from django.db import models, transaction +from django.db.models import Q +from django.utils import timezone + + +class FileMetaModel(models.Model): + """Model for File Meta""" + + value = models.JSONField() # 'path' and 'system' keys are always in value + created = models.DateTimeField(default=timezone.now) + last_updated = models.DateTimeField(auto_now=True) + + class Meta: + indexes = [ + models.Index( + models.F( + "value__system" + ), # Functional index on 'system' within 'value' + name="idx_value_system", + ), + models.Index( + models.F("value__path"), # Functional index on 'path' within 'value' + name="idx_value_path", + ), + ] + + def __str__(self): + return f"{self.value}" + + @classmethod + def create_or_update_file_meta(cls, value): + """ + Create or update a FileMetaModel instance based on the provided value dict containing 'system' and 'path'. + + Parameters: + - value (dict): A dictionary containing file metadata including required 'system' and 'path'. + + Returns: + - tuple (instance, created): The FileMetaModel instance and a boolean indicating if it was created (True) or updated (False). + """ + system = value.get("system") + path = value.get("path") + + # Use a transaction to ensure atomicity + with transaction.atomic(): + try: + file_meta = FileMetaModel.objects.select_for_update().get( + Q(value__system=system) & Q(value__path=path) + ) + file_meta.value = value + file_meta.save() + return file_meta, False + except FileMetaModel.DoesNotExist: + file_meta = FileMetaModel.objects.create(value=value) + return file_meta, True + + @classmethod + def get_by_path_and_system(cls, system, path): + """ + Retrieve file metadata by 'system' and 'path'. + + Raises: + - DoesNotExist: if file metadata entry not found + """ + return cls.objects.get(value__system=system, value__path=path) diff --git a/designsafe/apps/api/filemeta/tests.py b/designsafe/apps/api/filemeta/tests.py new file mode 100644 index 0000000000..b80fedafa8 --- /dev/null +++ b/designsafe/apps/api/filemeta/tests.py @@ -0,0 +1,161 @@ +import pytest +import json +from django.db import IntegrityError +from django.test import Client +from designsafe.apps.api.filemeta.models import FileMetaModel + + +@pytest.fixture +def filemeta_value_mock(): + value = {"system": "project-1234", "path": "/test/path.txt"} + return value + + +@pytest.fixture +def filemeta_db_mock(filemeta_value_mock): + system_id = filemeta_value_mock["system"] + path = filemeta_value_mock["path"] + value = filemeta_value_mock + file_meta_obj = FileMetaModel.objects.create(value=value) + return system_id, path, file_meta_obj + + +@pytest.fixture +def mock_access_success(mocker): + """Fixture to mock the listing function to always succeed.""" + mocker.patch("designsafe.apps.api.filemeta.views.listing") + + +@pytest.fixture +def mock_access_failure(mocker): + """Fixture to mock the listing function to always raise an exception.""" + mocker.patch( + "designsafe.apps.api.filemeta.views.listing", + side_effect=Exception("Access Denied"), + ) + + +@pytest.mark.django_db +def test_database_constraint(filemeta_value_mock): + FileMetaModel.objects.create(value=filemeta_value_mock) + with pytest.raises(IntegrityError) as excinfo: + FileMetaModel.objects.create(value=filemeta_value_mock) + assert "Unique" in str(excinfo.value) + + +@pytest.mark.django_db +def test_database_get_by_path_system(filemeta_db_mock, filemeta_value_mock): + FileMetaModel.get_by_path_and_system( + system=filemeta_value_mock["system"], path=filemeta_value_mock["path"] + ) + + with pytest.raises(FileMetaModel.DoesNotExist): + FileMetaModel.get_by_path_and_system(system="foo", path="bar/baz.txt") + + +@pytest.mark.django_db +def test_database_constraint(filemeta_value_mock): + FileMetaModel.objects.create(value=filemeta_value_mock) + with pytest.raises(IntegrityError) as excinfo: + FileMetaModel.objects.create(value=filemeta_value_mock) + assert "Unique" in str(excinfo.value) + + +@pytest.mark.django_db +def test_get_file_meta_unauthenticated(client, filemeta_db_mock, mock_access_success): + system_id, path, file_meta = filemeta_db_mock + response = client.get(f"/api/filemeta/{system_id}/{path}") + assert response.status_code == 401 + + +@pytest.mark.django_db +def test_get_file_meta( + client, authenticated_user, filemeta_db_mock, mock_access_success +): + system_id, path, file_meta = filemeta_db_mock + response = client.get(f"/api/filemeta/{system_id}/{path}") + assert response.status_code == 200 + + assert response.json() == { + "value": file_meta.value, + "name": "designsafe.file", + "lastUpdated": file_meta.last_updated.isoformat( + timespec="milliseconds" + ).replace("+00:00", "Z"), + } + + +@pytest.mark.django_db +def test_create_file_meta_no_access( + client, authenticated_user, filemeta_value_mock, mock_access_failure +): + response = client.post( + "/api/filemeta/", + data=json.dumps(filemeta_value_mock), + content_type="application/json", + ) + assert response.status_code == 403 + + +@pytest.mark.django_db +def test_create_file_meta_unauthenticated(client, filemeta_value_mock): + response = client.post( + "/api/filemeta/", + data=json.dumps(filemeta_value_mock), + content_type="application/json", + ) + assert response.status_code == 401 + + +@pytest.mark.django_db +def test_create_file_meta( + client, authenticated_user, filemeta_value_mock, mock_access_success +): + response = client.post( + "/api/filemeta/", + data=json.dumps(filemeta_value_mock), + content_type="application/json", + ) + assert response.status_code == 200 + + file_meta = FileMetaModel.objects.first() + assert file_meta.value == filemeta_value_mock + + +@pytest.mark.django_db +def test_create_file_meta_update_existing_entry( + client, + authenticated_user, + filemeta_db_mock, + filemeta_value_mock, + mock_access_success, +): + updated_value = {**filemeta_value_mock, "new_key": "new_value"} + + response = client.post( + "/api/filemeta/", + data=json.dumps(updated_value), + content_type="application/json", + ) + assert response.status_code == 200 + + file_meta = FileMetaModel.objects.first() + assert file_meta.value == updated_value + + +@pytest.mark.django_db +def test_create_file_metadata_missing_system_or_path( + client, + authenticated_user, + filemeta_db_mock, + filemeta_value_mock, + mock_access_success, +): + value_missing_system_path = {"foo": "bar"} + + response = client.post( + "/api/filemeta/", + data=json.dumps(value_missing_system_path), + content_type="application/json", + ) + assert response.status_code == 400 diff --git a/designsafe/apps/api/filemeta/urls.py b/designsafe/apps/api/filemeta/urls.py new file mode 100644 index 0000000000..a666913589 --- /dev/null +++ b/designsafe/apps/api/filemeta/urls.py @@ -0,0 +1,9 @@ +"""File Meta API routes""" + +from django.urls import path +from .views import FileMetaView, CreateFileMetaView + +urlpatterns = [ + path("/", FileMetaView.as_view(), name="filemeta-get"), + path("", CreateFileMetaView.as_view(), name="filemeta-create"), +] diff --git a/designsafe/apps/api/filemeta/views.py b/designsafe/apps/api/filemeta/views.py new file mode 100644 index 0000000000..d15dd49dd3 --- /dev/null +++ b/designsafe/apps/api/filemeta/views.py @@ -0,0 +1,100 @@ +"""File Meta view""" +import logging +import json +from django.http import JsonResponse, HttpRequest +from designsafe.apps.api.datafiles.operations.agave_operations import listing +from designsafe.apps.api.exceptions import ApiException +from designsafe.apps.api.filemeta.models import FileMetaModel +from designsafe.apps.api.views import AuthenticatedApiView + + +logger = logging.getLogger(__name__) + + +def check_access(request, system_id: str, path: str, check_for_writable_access=False): + """ + Check if the user has access to a specific system and path. + + This function utilizes the listing functionality to verify access. Writable access is only given for files + in DS project systems. + + Raises: + - ApiException: If the user is forbidden from accessing or modifying the metadata. + """ + + if check_for_writable_access and not system_id.startswith("project-"): + error_msg = f"Metadata updates are not allowed on non-project systems (system={system_id})." + logger.error(error_msg) + raise ApiException(error_msg, status=403) + + try: + # TODO_V3 update to use renamed (i.e. "tapis") client + listing(request.user.agave_oauth.client, system_id, path) + except Exception as exc: # pylint:disable=broad-exception-caught + logger.error( + f"user cannot access any related metadata as listing failed for {system_id}/{path} with error {str(exc)}." + ) + raise ApiException("User forbidden to access metadata", status=403) from exc + + +# TODO_V3 update to allow JWT access DES-2706: https://github.com/DesignSafe-CI/portal/pull/1192 +class FileMetaView(AuthenticatedApiView): + """View for creating and getting file metadata""" + + def get(self, request: HttpRequest, system_id: str, path: str): + """Return metadata for system_id/path + + If no metadata for system_id and path, then empty dict is returned + """ + check_access(request, system_id, path) + + result = {} + try: + logger.debug(f"Get file metadata. system:{system_id} path:{path}") + file_meta = FileMetaModel.get_by_path_and_system( + system=system_id, path=path + ) + result = { + "value": file_meta.value, + "lastUpdated": file_meta.last_updated, + "name": "designsafe.file", + } + except FileMetaModel.DoesNotExist: + pass + + return JsonResponse(result, safe=False) + + +# TODO_V3 update to allow JWT access DES-2706: https://github.com/DesignSafe-CI/portal/pull/1192 +class CreateFileMetaView(AuthenticatedApiView): + """View for creating (and updating) file metadata""" + + def post(self, request: HttpRequest): + """Create metadata for system_id/path.""" + + value = json.loads(request.body) + if "system" not in value or "path" not in value: + logger.error( + f"Unable to create or update file metadata as system and path not in payload: {value}" + ) + raise ApiException("System and path are required in payload", status=400) + + system_id = value["system"] + path = value["path"] + + check_access(request, system_id, path, check_for_writable_access=True) + + try: + logger.info( + f"Creating or updating file metadata. system:{system_id} path:{path}" + ) + + FileMetaModel.create_or_update_file_meta(value) + return JsonResponse({"result": "OK"}) + except Exception as exc: + logger.exception( + f"Unable to create or update file metadata: {system_id}/{path}" + ) + raise ApiException( + "Unable to create or update file metadata", status=500 + ) from exc diff --git a/designsafe/apps/api/publications_v2/apps.py b/designsafe/apps/api/publications_v2/apps.py index 6745d5e052..512376f288 100644 --- a/designsafe/apps/api/publications_v2/apps.py +++ b/designsafe/apps/api/publications_v2/apps.py @@ -1,9 +1,9 @@ -"""Projects V2 API""" +"""Publications V2 API""" from django.apps import AppConfig -class ProjectsV2AppConfig(AppConfig): - """App config for Projects V2 API""" +class PublicationsV2AppConfig(AppConfig): + """App config for Publications V2 API""" name = "designsafe.apps.api.publications_v2" label = "publications_v2_api" diff --git a/designsafe/apps/api/urls.py b/designsafe/apps/api/urls.py index c45605fcac..7af0b1dc91 100644 --- a/designsafe/apps/api/urls.py +++ b/designsafe/apps/api/urls.py @@ -16,6 +16,8 @@ url(r'^datafiles/', include('designsafe.apps.api.datafiles.urls')), url(r'^publications/', include('designsafe.apps.api.publications.urls')), + url(r'^filemeta/', include('designsafe.apps.api.filemeta.urls')), + url(r'^logger/$', LoggerApi.as_view(), name='logger'), url(r'^notifications/', include('designsafe.apps.api.notifications.urls')), url(r'^users/', include('designsafe.apps.api.users.urls')), diff --git a/designsafe/apps/api/views.py b/designsafe/apps/api/views.py index dffe31a90e..1ee0eb4d0c 100644 --- a/designsafe/apps/api/views.py +++ b/designsafe/apps/api/views.py @@ -1,4 +1,4 @@ -from django.http.response import HttpResponse, HttpResponseForbidden +from django.http.response import HttpResponse, HttpResponseForbidden, JsonResponse from django.views.generic import View from requests.exceptions import ConnectionError, HTTPError from .exceptions import ApiException @@ -51,6 +51,16 @@ def dispatch(self, request, *args, **kwargs): status=status, content_type='application/json') +class AuthenticatedApiView(BaseApiView): + + def dispatch(self, request, *args, **kwargs): + """Returns 401 if user is not authenticated.""" + + if not request.user.is_authenticated: + return JsonResponse({"message": "Unauthenticated user"}, status=401) + return super(AuthenticatedApiView, self).dispatch(request, *args, **kwargs) + + class LoggerApi(BaseApiView): """ Logger API for capturing logs from the front-end. diff --git a/designsafe/settings/common_settings.py b/designsafe/settings/common_settings.py index baa94d97bd..6c2c96760e 100644 --- a/designsafe/settings/common_settings.py +++ b/designsafe/settings/common_settings.py @@ -87,6 +87,7 @@ 'designsafe.apps.api.datafiles', 'designsafe.apps.api.projects_v2', 'designsafe.apps.api.publications_v2', + 'designsafe.apps.api.filemeta', 'designsafe.apps.accounts', 'designsafe.apps.cms_plugins', 'designsafe.apps.box_integration', diff --git a/designsafe/settings/test_settings.py b/designsafe/settings/test_settings.py index e88bd5e0be..44b03240c7 100644 --- a/designsafe/settings/test_settings.py +++ b/designsafe/settings/test_settings.py @@ -85,6 +85,7 @@ 'designsafe.apps.api', 'designsafe.apps.api.notifications', 'designsafe.apps.api.projects_v2', + 'designsafe.apps.api.filemeta', 'designsafe.apps.accounts', 'designsafe.apps.cms_plugins', 'designsafe.apps.box_integration',