Merge pull request #398 from DigitalSlideArchive/assetstore-import

Import data if imported from an assetstore

Dockerfile

@@ -18,6 +18,7 @@ RUN apt-get update && \
     ca-certificates \
     curl \
     fonts-dejavu \
+    fuse \
     git \
     less \
     libmagic-dev \
@@ -93,6 +94,8 @@ RUN curl -sL https://deb.nodesource.com/setup_14.x | bash && \
     nodejs && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
+RUN mkdir -p /fuse --mode=a+rwx
+
 RUN mkdir -p wsi_deid && \
     mkdir -p /conf
 
@@ -102,10 +105,11 @@ COPY . .
 
 # By using --no-cache-dir the Docker image is smaller
 RUN python -m pip install --no-cache-dir \
+    git+https://github.com/DigitalSlideArchive/import-tracker.git \
     # git+https://github.com/DigitalSlideArchive/DSA-WSI-DeID.git \
     . \
     # girder[mount] adds dependencies to show tiles from S3 assets \
-    # girder[mount] \
+    girder[mount] \
     # Add additional girder plugins here \
     # girder-homepage \
     # Use prebuilt wheels whenever possible \
@@ -132,4 +136,4 @@ COPY ./devops/wsi_deid/girder.local.conf ./devops/wsi_deid/provision.py ./devops
 
 ENTRYPOINT ["/usr/bin/tini", "--"]
 
-CMD python /conf/provision.py && girder serve
+CMD python /conf/provision.py && (girder mount /fuse 2>/dev/null || true) && girder serve

devops/wsi_deid/docker-compose.yml

@@ -8,6 +8,22 @@ services:
     # so that local file assetstores and logs are owned by yourself.
     # user: ${CURRENT_UID}
     restart: unless-stopped
+    # To have full capabilities with S3 assetstores, we use a user file system
+    # (fuse).  This requires some privileges.  This is not needed if only
+    # filesystem assetstores are used.  Instead of privileged mode, fuse can
+    # use specific devices, security_opt, and cap_add:
+    # devices:
+    #   - /dev/fuse:/dev/fuse
+    # security_opt:
+    #   - apparmor:unconfined
+    # cap_add:
+    #   - SYS_ADMIN
+    # but these may be somewhat host specific, so we default to privileged.  If
+    # the docker daemon is being run with --no-new-privileges, fuse may not
+    # work.
+    # See also https://github.com/docker/for-linux/issues/321 for possible
+    # methods to avoid both privileged mode and cap_add SYS_ADMIN.
+    privileged: true
     # Set DSA_PORT to expose the interface on another port (default 8080).
     ports:
       - "${DSA_PORT:-8080}:8080"

devops/wsi_deid/install_and_start_isyntax.sh

@@ -22,4 +22,5 @@ set -e
 ) || true
 # This is our original start method
 python /conf/provision.py
+girder mount /fuse || true
 girder serve

wsi_deid/__init__.py

@@ -3,7 +3,7 @@
 import girder
 import PIL.Image
 import psutil
-from girder import plugin
+from girder import events, plugin
 from girder.constants import AssetstoreType
 from girder.exceptions import GirderException, ValidationException
 from girder.models.assetstore import Assetstore
@@ -13,6 +13,7 @@
 from girder.utility import setting_utilities
 from pkg_resources import DistributionNotFound, get_distribution
 
+from . import assetstore_import
 from .constants import PluginSettings
 from .import_export import SftpMode
 from .rest import WSIDeIDResource, addSystemEndpoints
@@ -106,3 +107,6 @@ def load(self, info):
         if idx1 not in File()._indices:
             File().ensureIndex(idx1)
         PIL.Image.ANTIALIAS = PIL.Image.LANCZOS
+
+        events.bind('rest.post.assetstore/:id/import.after', 'wsi_deid',
+                    assetstore_import.assetstoreImportEvent)

wsi_deid/assetstore_import.py

@@ -0,0 +1,87 @@
+import os
+
+from girder import logger
+from girder.api.rest import getCurrentUser
+from girder.models.folder import Folder
+from girder.models.item import Item
+from girder.models.setting import Setting
+from girder.utility.progress import ProgressContext
+
+from . import import_export
+from .constants import PluginSettings
+
+
+def assetstoreImportEvent(event):
+    """
+    After an assetstore import, check if the import was done within the Unfiled
+    folder.  If so, import the data.
+    """
+    logger.info('Processing assetstore import event')
+    check = False
+    try:
+        params = event.info['returnVal']['params']
+        if params['destinationType'] == 'folder':
+            folder = Folder().load(params['destinationId'], force=True)
+            if import_export.isProjectFolder(folder) == 'unfiled':
+                check = True
+    except Exception:
+        logger.exception('Processing assetstore import event: Failed to check')
+        return
+    if not check:
+        logger.info('Processing assetstore import event: not the unfiled folder')
+        return
+    logger.info('Processing assetstore import event: will process')
+    progress = True
+    user = getCurrentUser()
+    with ProgressContext(progress, user=user, title='Importing data') as ctx:
+        importFromUnfiled(user, ctx)
+
+
+def assetstoreIngestFindFilesFolder(folder, excelFiles, imageFiles):
+    """
+    Walk a folder and add likely files to excel and image file lists.
+
+    :param folder: folder to walk
+    :param excelFiles: list of items that are probably excel files.
+    :param imageFiles: list of items that are probably image files.
+    """
+    for subfolder in Folder().childFolders(folder, parentType='folder'):
+        assetstoreIngestFindFilesFolder(subfolder, excelFiles, imageFiles)
+    for item in Folder().childItems(folder):
+        _, ext = os.path.splitext(item['name'])
+        # skip this item if it has metadata
+        if len(item.get('meta', {})):
+            continue
+        if ext.lower() in {'.xls', '.xlsx', '.csv'} and not item['name'].startswith('~$'):
+            if len(list(Item().childFiles(item, limit=1))):
+                excelFiles.append(item)
+        elif 'largeImage' in item:
+            imageFiles.append(item)
+
+
+def assetstoreIngestFindFiles(importPath):
+    """
+    Get a list of excel and image files for import.
+
+    :param importPath: the import folder path (ignored).
+    :returns: a two tuple of lists of excel items and image items.
+    """
+    unfiledFolderId = Setting().get(PluginSettings.WSI_DEID_UNFILED_FOLDER)
+    unfiledFolder = Folder().load(unfiledFolderId, force=True, exc=True)
+
+    excelFiles = []
+    imageFiles = []
+    assetstoreIngestFindFilesFolder(unfiledFolder, excelFiles, imageFiles)
+    return excelFiles, imageFiles
+
+
+def importFromUnfiled(user, ctx):
+    """
+    Walk the unfiled folder.  Collect all excel and image files that haven't
+    been processed before, then process them as if they came through the import
+    mechanism.
+
+    :param user: the user that started the process.
+    :param ctx: A progress context.
+    """
+    import_export.ingestData(user=user, walkData=assetstoreIngestFindFiles, ctx=ctx)