Merge pull request #363 from RockefellerArchiveCenter/v1.6

v1.6

aurora/.gitignore

@@ -1,5 +1,4 @@
 *.pyc
 test_scripts/
-static/
 .coverage
 .DS_Store

aurora/bag_transfer/accession/models.py

@@ -26,8 +26,13 @@ class Accession(models.Model):
     last_modified = models.DateTimeField(auto_now=True)
     CREATED = 10
     DELIVERED = 20
+    RECORD_CREATED = 30
+    COMPLETE = 40
     PROCESS_STATUS_CHOICES = (
         (CREATED, 'Accession created'),
-        (DELIVERED, 'Accession delivered to queue'),
+        (DELIVERED, 'Accession transfers delivered to queue'),
+        (RECORD_CREATED, 'Accession record created in ArchivesSpace'),
+        (COMPLETE, 'Accession complete')
     )
     process_status = models.PositiveSmallIntegerField(choices=PROCESS_STATUS_CHOICES, default=10, null=True, blank=True)
+    archivesspace_identifier = models.CharField(max_length=255, blank=True, null=True)

aurora/bag_transfer/accession/urls.py

@@ -3,6 +3,7 @@
 
 urlpatterns = [
     url(r'^$', 	AccessionView.as_view(), name='list'),
-    url(r'^add/', AccessionRecordView.as_view(), name='detail'),
+    url(r'^(?P<pk>\d+)/$', AccessionDetailView.as_view(), name='detail'),
+    url(r'^add/', AccessionCreateView.as_view(), name='add'),
     url(r'^saved-datatable/$', SavedAccessionsDatatableView.as_view(), name='saved-datatable'),
 ]

aurora/bag_transfer/accession/views.py

@@ -5,20 +5,21 @@
 import json
 import requests
 
-from django.views.generic import ListView, View
+from django.views.generic import DetailView, ListView, View
 from django_datatables_view.base_datatable_view import BaseDatatableView
 from django.db.models import CharField, F
 from django.db.models.functions import Concat
 from django.contrib import messages
 from django.http import HttpResponse
-from django.shortcuts import render, redirect
+from django.shortcuts import render, redirect, reverse
 
 from aurora import settings
 from bag_transfer.accession.models import Accession
 from bag_transfer.accession.forms import AccessionForm, CreatorsFormSet
 from bag_transfer.accession.db_functions import GroupConcat
 from bag_transfer.api.serializers import AccessionSerializer
 from bag_transfer.lib.clients import ArchivesSpaceClient
+from bag_transfer.lib.view_helpers import file_size
 from bag_transfer.mixins.formatmixins import JSONResponseMixin
 from bag_transfer.mixins.authmixins import ArchivistMixin, AccessioningArchivistMixin
 from bag_transfer.models import Archives, RecordCreators, Organization, BAGLog, LanguageCode
@@ -65,7 +66,12 @@ def handle_ajax_request(self, request):
         return self.render_to_json_response(rdata)
 
 
-class AccessionRecordView(AccessioningArchivistMixin, JSONResponseMixin, View):
+class AccessionDetailView(AccessioningArchivistMixin, DetailView):
+    template_name = 'accession/detail.html'
+    model = Accession
+
+
+class AccessionCreateView(AccessioningArchivistMixin, JSONResponseMixin, View):
     template_name = "accession/create.html"
     model = Accession
     form_class = AccessionForm
@@ -106,7 +112,7 @@ def post(self, request, *args, **kwargs):
                 except Exception as e:
                     print(e)
                     messages.error(request, 'Error delivering accession data: {}'.format(e))
-            return redirect('accession:list')
+            return redirect('accession:detail', accession.pk)
         messages.error(
             request,
             "There was a problem with your submission. Please correct the error(s) below and try again.")
@@ -210,27 +216,36 @@ def handle_ajax_request(self, request):
 
 class SavedAccessionsDatatableView(ArchivistMixin, BaseDatatableView):
     model = Accession
-    columns = ['title', 'created', 'extent_files', 'extent_size']
-    order_columns = ['title', 'created', 'extent_files', 'extent_size']
+    columns = ['title', 'created', 'extent_files', 'accession_transfers__machine_file_identifier', 'extent_size']
+    order_columns = ['title', 'created', 'extent_files', 'accession_transfers__machine_file_identifier', 'extent_size']
     max_display_length = 500
 
     def get_filter_method(self): return self.FILTER_ICONTAINS
 
+    def title(self, accession):
+        return "{} ({})".format(accession.title, accession.accession_number) if accession.accession_number else accession.title
+
+    def transfers(self, accession):
+        transfers = accession.accession_transfers.count()
+        label = 'transfer' if transfers == 1 else 'transfers'
+        return "{} {}".format(transfers, label)
+
     def button(self, accession):
         button = "Accession not delivered"
         if self.request.user.can_accession():
             button = '<a href="#" class="btn btn-primary pull-right deliver">Deliver Accession</a>' \
                 if (accession.process_status < Accession.DELIVERED) \
-                else '<p class="pull-right" style="margin-right:.7em;">Accession delivered</p>'
+                else '<p class="pull-right" style="margin-right:.7em;">'+accession.get_process_status_display()+'</p>'
         return button
 
     def prepare_results(self, qs):
         json_data = []
         for accession in qs:
             json_data.append([
-                accession.title,
+                "<a href='{}'>{}</a.".format(reverse('accession:detail', kwargs={'pk': accession.pk}), self.title(accession)),
                 accession.created.astimezone(tz.tzlocal()).strftime('%b %e, %Y %I:%M %p'),
-                "{} files ({})".format(accession.extent_files, accession.extent_size),
+                "{} files ({})".format(accession.extent_files, file_size(int(accession.extent_size))),
+                self.transfers(accession),
                 self.button(accession),
                 accession.pk
             ])

aurora/bag_transfer/api/serializers.py

@@ -243,11 +243,11 @@ class Meta:
 
 
 class AccessionSerializer(serializers.HyperlinkedModelSerializer):
-    creators = RecordCreatorsSerializer(many=True)
-    transfers = ArchivesListSerializer(source='accession_transfers', many=True)
-    organization = serializers.StringRelatedField()
-    rights_statements = RightsStatementSerializer(many=True)
-    language = serializers.StringRelatedField()
+    creators = RecordCreatorsSerializer(many=True, read_only=True)
+    transfers = ArchivesListSerializer(source='accession_transfers', many=True, read_only=True)
+    organization = serializers.StringRelatedField(read_only=True)
+    rights_statements = RightsStatementSerializer(many=True, read_only=True)
+    language = serializers.StringRelatedField(read_only=True)
 
     class Meta:
         model = Accession

aurora/bag_transfer/lib/files_helper.py

@@ -3,8 +3,7 @@
 import datetime
 import tarfile
 import zipfile
-from distutils.dir_util import copy_tree
-from shutil import rmtree, move
+from shutil import rmtree, move, copytree
 import psutil
 import pwd
 
@@ -20,7 +19,10 @@ def open_files_list():
     path_list = []
 
     for proc in psutil.process_iter():
-        open_files = proc.open_files()
+        try:
+            open_files = proc.open_files()
+        except psutil.AccessDenied as e:
+            print proc.as_dict(), e
         if open_files:
             for fileObj in open_files:
                 path_list.append(fileObj.path)
@@ -164,11 +166,13 @@ def tar_extract_all(file_path, tmp_dir):
 
     return extracted
 
-def dir_extract_all(file_path,tmp_dir):
+def dir_extract_all(file_path, tmp_dir):
     extracted = False
     try:
         # notice forward slash missing
-        copy_tree(file_path,'{}{}'.format(tmp_dir, file_path.split('/')[-1]), update=1)
+        if is_dir_or_file('{}{}'.format(tmp_dir, file_path.split('/')[-1])):
+            rmtree('{}{}'.format(tmp_dir, file_path.split('/')[-1]))
+        copytree(file_path, '{}{}'.format(tmp_dir, file_path.split('/')[-1]))
         extracted = True
     except Exception as e:
         print e

aurora/bag_transfer/lib/view_helpers.py

@@ -0,0 +1,14 @@
+def file_size(num):
+    for unit in ['B', 'KB', 'MB', 'GB']:
+        if abs(num) < 1024.0:
+            return "%3.1f %s" % (num, unit)
+        num /= 1024.0
+
+
+def label_class(status):
+    label_class = 'green'
+    if status in [10, 20]:
+        label_class = 'yellow'
+    elif status in [30, 60]:
+        label_class = 'red'
+    return label_class

aurora/bag_transfer/migrations/0026_auto_20190918_2154.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.22 on 2019-09-19 01:54
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bag_transfer', '0025_auto_20190812_1455'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='accession',
+            name='archivesspace_identifier',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='accession',
+            name='process_status',
+            field=models.PositiveSmallIntegerField(blank=True, choices=[(10, 'Accession created'), (20, 'Accession transfers delivered to queue'), (30, 'Accession record created in ArchivesSpace'), (40, 'Accession complete')], default=10, null=True),
+        ),
+    ]

aurora/bag_transfer/models.py

@@ -382,7 +382,7 @@ def save_mtm_fields(self, cls, field, model_field, metadata):
                 for f in metadata[field]:
                     new_obj = cls.objects.get_or_create(**{model_field: f})[0]
                     obj_list.append(new_obj)
-            else:
+            elif len(metadata[field].strip()):
                 new_obj = cls.objects.get_or_create(**{model_field: metadata[field]})[0]
                 obj_list.append(new_obj)
         return obj_list

aurora/bag_transfer/signals.py

@@ -7,6 +7,7 @@
 from django.db.models.signals import m2m_changed, pre_delete, post_delete, post_save
 from django.dispatch import receiver
 
+from bag_transfer.accession.models import Accession
 from bag_transfer.lib.files_helper import chown_path_to_root
 from bag_transfer.lib.RAC_CMD import delete_system_group
 from bag_transfer.models import Archives, User, BagInfoMetadata, Organization, DashboardMonthData, DashboardRecordTypeData
@@ -29,8 +30,12 @@ def set_is_staff(sender, instance, **kwargs):
 
 
 @receiver(post_save, sender=Archives)
-@receiver(post_delete, sender=Archives)
+@receiver(pre_delete, sender=Archives)
 def dashboard_data(sender, instance, **kwargs):
+    """
+    Updates dashboard data each time a transfer is saved or deleted, which
+    avoids expensive data operations on the database.
+    """
     today = date.today()
     current = today - relativedelta(years=1)
     if instance.process_status >= sender.TRANSFER_COMPLETED:
@@ -61,3 +66,17 @@ def dashboard_data(sender, instance, **kwargs):
                 )[0]
                 data.count = Archives.objects.filter(organization=organization, metadata__record_type=label).count()
                 data.save()
+
+
+@receiver(post_save, sender=Archives)
+def update_accession_status(sender, instance, **kwargs):
+    """
+    Updates Accession status to COMPLETE if all of the transfers in an accession
+    have finished processing.
+    """
+    if instance.process_status == Archives.ACCESSIONING_COMPLETE:
+        accession = instance.accession
+        last_update = sorted(set([t.process_status for t in accession.accession_transfers.all()]))[0]
+        if last_update == Archives.ACCESSIONING_COMPLETE:
+            accession.process_status = Accession.COMPLETE
+            accession.save()