Merge branch 'edge'

.eslintrc

@@ -0,0 +1,12 @@
+{
+    "parserOptions": {
+        "ecmaVersion": 6,
+        "sourceType": "module",
+    },
+    "env": {
+        "browser": true,
+    },
+    "rules": {
+        "semi": 2
+    }
+}

bin/run-celery.sh

@@ -2,4 +2,5 @@
 
 # Run Celery as the DesignSafe Community Account
 su tg458981 -c "celery -A designsafe beat -l info --pidfile= --schedule=/tmp/celerybeat-schedule" &
-su tg458981 -c "celery -A designsafe worker -l info --autoscale=10,2"
+su tg458981 -c "celery -A designsafe worker -l info --autoscale=15,5 -Q indexing,files" &
+su tg458981 -c "celery -A designsafe worker -l info --autoscale=10,3 -Q default,api"

bower.json

@@ -35,8 +35,12 @@
     "angular-drag-and-drop-lists": "1.4.0",
     "angular-xeditable": "0.1.12",
     "angular-httpi": "*",
+    "angular-slick-carousel": "^3.1.7",
+    "exif-js": "^2.1.1",
     "d3": "^4.4.0",
-    "angular-slick-carousel": "^3.1.7"
+    "angular-native-dragdrop": "^1.2.2",
+    "leaflet-measure": "^2.1.7",
+    "tether": "^1.4.0"
   },
   "resolutions": {
     "angular": "~1.4.8",

conf/nginx.conf

@@ -52,6 +52,10 @@ http {
         ssl_protocols               TLSv1 TLSv1.1 TLSv1.2;
         ssl_ciphers                 "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4";
 
+        location /uploads {
+            alias /corral-repl/tacc/NHERI/uploads;
+        }
+
         location /media  {
             alias /var/www/designsafe-ci.org/media;
         }

designsafe.env.sample

@@ -78,12 +78,24 @@ AGAVE_TENANT_BASEURL=https://agave.designsafe-ci.org
 AGAVE_CLIENT_KEY=
 AGAVE_CLIENT_SECRET=
 AGAVE_TOKEN_SESSION_ID=
+AGAVE_JWT_PUBKEY=
 
 # Long-lived token used by the portal for "admin" actions; obtain from API store interface
 AGAVE_SUPER_TOKEN=
+AGAVE_JWT_ISSUER=
+AGAVE_JWT_HEADER=
+AGAVE_JWT_USER_CLAIM_FIELD=
 
 AGAVE_STORAGE_SYSTEM=designsafe.storage.default
 
+PROJECT_SYSTEM_STORAGE_CREDENTIALS=
+
+BOX_APP_CLIENT_ID=
+BOX_APP_CLIENT_SECRET=
+
+DROPBOX_APP_KEY=
+DROPBOX_APP_SECRET=
+
 # djangoRT
 RT_HOST=https://hostname.example.com/REST/1.0/
 RT_USERNAME=username
@@ -100,3 +112,27 @@ NEES_DATABASE_HOST=
 NEES_DATABASE_PORT=
 NEES_DATABASE_USER=
 NEES_DATABASE_PASSWORD=
+
+
+# FOR CMS/RECAPTCHA
+DJANGOCMS_FORMS_RECAPTCHA_PUBLIC_KEY=
+DJANGOCMS_FORMS_RECAPTCHA_SECRET_KEY=
+
+FLOWER_BROKER=
+FLOWER_BROKER_API=
+
+#CELERY config
+
+BROKER_URL_PROTOCOL=amqp://
+BROKER_URL_USERNAME= 
+BROKER_URL_PWD= 
+BROKER_URL_HOST= 
+BROKER_URL_PORT= 
+BROKER_URL_VHOST= 
+#BROKER_URL = 'amqp://designsafe:pwd@rabbitmq:5672//'
+CELERY_RESULT_BACKEND_PROTOCOL= 
+CELERY_RESULT_BACKEND_USERNAME= 
+CELERY_RESULT_BACKEND_PWD= 
+CELERY_RESULT_BACKEND_HOST= 
+CELERY_RESULT_BACKEND_PORT= 
+CELERY_RESULT_BACKEND_DB= 

designsafe/apps/accounts/templates/designsafe/apps/accounts/base.html

@@ -26,7 +26,7 @@ <h1 class="headline headline-research">{{title}}</h1>
           <li {% if request.path == item_url %}class="active"{% endif %}><a href="{{item_url}}">Licenses</a></li>
 
           {% url 'designsafe_accounts:manage_notifications' as item_url %}
-          <li {% if request.path == item_url %}class="active"{% endif %}><a href="{{item_url}}">Notifications</a></li>
+          <li {% if request.path == item_url %}class="active"{% endif %}><a href="{{item_url}}">Notification Settings</a></li>
 
           {% url 'designsafe_accounts:manage_applications' as item_url %}
           <li {% if request.path == item_url %}class="active"{% endif %}><a href="{{item_url}}">3rd Party Apps</a></li>

designsafe/apps/api/agave/filemanager/agave.py

@@ -28,6 +28,8 @@ class AgaveFileManager(BaseFileManager):
          'path': '/corral-repl/tacc/NHERI/shared'},
         {'regex': r'^designsafe.storage.community$',
          'path': '/corral-repl/tacc/NHERI/community'},
+        {'regex': r'^designsafe.storage.published$',
+         'path': '/corral-repl/tacc/NHERI/published'},
         {'regex': r'^project\-',
          'path': '/corral-repl/tacc/NHERI/projects'}
     ]
@@ -53,7 +55,8 @@ def import_data(self, system, file_path, from_system, from_file_path):
         res = f.import_data(from_system, from_file_path)
         file_name = from_file_path.split('/')[-1]
         reindex_agave.apply_async(kwargs={'username': 'ds_admin',
-                                          'file_id': '{}/{}'.format(system, os.path.join(file_path, file_name))})
+                                          'file_id': '{}/{}'.format(system, os.path.join(file_path, file_name))},
+                                  queue='indexing')
         return res
 
     def copy(self, system, file_path, dest_path=None, dest_name=None):
@@ -75,7 +78,8 @@ def copy(self, system, file_path, dest_path=None, dest_name=None):
 
         # schedule celery task to index new copy
         reindex_agave.apply_async(kwargs = {'username': 'ds_admin',
-                                            'file_id': '{}/{}/{}'.format(system, dest_path.strip('/'), dest_name)})
+                                            'file_id': '{}/{}/{}'.format(system, dest_path.strip('/'), dest_name)},
+                                  queue='indexing')
 
         return copied_file
 
@@ -84,7 +88,8 @@ def delete(self, system, path):
         parent_path = '/'.join(path.strip('/').split('/')[:-1])
         reindex_agave.apply_async(kwargs = {'username': 'ds_admin',
                                             'file_id': '{}/{}'.format(system, parent_path),
-                                            'levels': 1})
+                                            'levels': 1}, 
+                                            queue='indexing')
         return resp
 
     def download(self, system, path):
@@ -109,7 +114,8 @@ def mkdir(self, system, file_path, dir_name):
         f = BaseFileResource(self._ag, system, file_path)
         resp = f.mkdir(dir_name)
         reindex_agave.apply_async(kwargs = {'username': 'ds_admin',
-                                            'file_id': '{}/{}'.format(system, file_path)})
+                                            'file_id': '{}/{}'.format(system, file_path)},
+                                            queue='indexing')
         return resp
 
     def move(self, system, file_path, dest_path, dest_name=None):
@@ -119,10 +125,12 @@ def move(self, system, file_path, dest_path, dest_name=None):
         parent_path = parent_path.strip('/') or '/'
         reindex_agave.apply_async(kwargs = {'username': 'ds_admin',
                                             'file_id': '{}/{}'.format(system, parent_path),
-                                            'levels': 1})
+                                            'levels': 1},
+                                            queue='indexing')
         reindex_agave.apply_async(kwargs = {'username': 'ds_admin',
                                             'file_id': '{}/{}'.format(system, os.path.join(dest_path, resp.name)),
-                                            'levels': 1})
+                                            'levels': 1},
+                                            queue='indexing')
         return resp
 
     def rename(self, system, file_path, rename_to):
@@ -131,7 +139,8 @@ def rename(self, system, file_path, rename_to):
         parent_path = '/'.join(file_path.strip('/').split('/')[:-1])
         reindex_agave.apply_async(kwargs = {'username': 'ds_admin',
                                             'file_id': '{}/{}'.format(system, parent_path),
-                                            'levels': 1})
+                                            'levels': 1},
+                                            queue='indexing')
         return resp
 
     def share(self, system, file_path, username, permission):
@@ -142,7 +151,8 @@ def share(self, system, file_path, username, permission):
         pem.permission_bit = permission
         resp = pem.save()
         reindex_agave.apply_async(kwargs = {'username': 'ds_admin',
-                                            'file_id': '{}/{}'.format(system, file_path)})
+                                            'file_id': '{}/{}'.format(system, file_path)},
+                                            queue='indexing')
         return resp
 
     def trash(self, system, file_path, trash_path):
@@ -171,16 +181,19 @@ def trash(self, system, file_path, trash_path):
         parent_path = parent_path.strip('/') or '/'
         reindex_agave.apply_async(kwargs = {'username': 'ds_admin',
                                             'file_id': '{}/{}'.format(system, trash_path),
-                                            'levels': 1})
+                                            'levels': 1},
+                                            queue='indexing')
         reindex_agave.apply_async(kwargs = {'username': 'ds_admin',
                                             'file_id': '{}/{}'.format(system, parent_path),
-                                            'levels': 1})
+                                            'levels': 1},
+                                            queue='indexing')
         return resp
 
     def upload(self, system, file_path, upload_file):
         f = BaseFileResource(self._ag, system, file_path)
         resp = f.upload(upload_file)
         reindex_agave.apply_async(kwargs = {'username': 'ds_admin',
                                             'file_id': '{}/{}'.format(system, file_path),
-                                            'levels': 1})
+                                            'levels': 1},
+                                            queue='indexing')
         return resp

designsafe/apps/api/agave/filemanager/public_search_index.py

@@ -12,6 +12,7 @@
 import os
 import re
 import datetime
+import itertools
 from django.conf import settings
 from elasticsearch import TransportError
 from elasticsearch_dsl import Search, DocType
@@ -38,6 +39,120 @@
 except KeyError as e:
     logger.exception('ELASTIC_SEARCH missing %s' % e)
 
+class PublicationIndexed(DocType):
+    class Meta:
+        index = 'published'
+        doc_type = 'publication'
+
+class Publication(object):
+    def __init__(self, wrap=None, project_id=None, *args, **kwargs):
+        if wrap is not None:
+            if isinstance(wrap, PublicationIndexed):
+                self._wrap = wrap
+            else:
+                s = PublicationIndexed.search()
+                s.query = Q({"term":
+                              {"projectId._exact": wrap['projectId']}
+                            })
+                try:
+                    res = s.execute()
+                except TransportError as e:
+                    if e.status_code != 404:
+                        res = s.execute()
+
+                if res.hits.total:
+                    self._wrap = res[0]
+                    raise Exception('Initializing from existent publication '
+                                    'and a publication object was given. '
+                                    'Are you sure you want to do this? ')
+                else: 
+                    self._wrap = PublicationIndexed(**wrap)
+
+        elif project_id is not None:
+            s = PublicationIndexed.search()
+            s.query = Q({"term": {"projectId._exact": project_id }})
+            logger.debug('p serach query: {}'.format(s.to_dict()))
+            try:
+                res = s.execute()
+            except TransportError as e:
+                if e.status_code == 404:
+                    raise
+                res = s.execute()
+
+            if res.hits.total:
+                self._wrap = res[0]
+            else:
+                self._wrap = PublicationIndexed(projectId=project_id)
+        else:
+            self._wrap = PublicationIndexed()
+
+    @classmethod
+    def listing(cls):
+        list_search = PublicSearchManager(cls,
+                                          PublicationIndexed.search(),
+                                          page_size=100)
+        list_search._search.query = Q({"match_all":{}})
+        list_search.sort({'projectId._exact': 'asc'})
+        #s = PublicationIndexed.search()
+        #s.query = Q({"match_all":{}})
+        #try:
+        #    res = s.execute()
+        #except TransportError as err:
+        #    if err.satus_code == 404:
+        #        raise
+        #    res = s.execute()
+
+        return list_search.results(0)
+
+    @property
+    def id(self):
+        return self._wrap.meta.id
+
+    def save(self, **kwargs):
+        self._wrap.save(**kwargs)
+        return self
+
+    def to_dict(self):
+        return self._wrap.to_dict()
+
+    def to_file(self):
+        dict_obj = {'agavePath': 'agave://designsafe.storage.published/{}'.\
+                                 format(self.project.value.projectId),
+                     'children': [],
+                     'deleted': False,
+                     'format': 'folder',
+                     'length': 24731027,
+                     'meta': {
+                         'title': self.project['value']['title']
+                     },
+                     'name': self.project.value.projectId,
+                     'path': '/{}'.format(self.project.value.projectId),
+                     'permissions': 'READ',
+                     'project': self.project.value.projectId,
+                     'system': 'designsafe.storage.published',
+                     'systemId': 'designsafe.storage.published',
+                     'type': 'dir'}
+        return dict_obj
+
+    def related_file_paths(self):
+        dict_obj = self._wrap.to_dict()
+        related_objs = dict_obj.get('modelConfigs', []) + \
+                       dict_obj.get('analysisList', []) + \
+                       dict_obj.get('sensorsList', []) + \
+                       dict_obj.get('eventsList', [])
+        file_paths = []
+        proj_sys = 'project-{}'.format(dict_obj['project']['uuid'])
+        for obj in related_objs:
+            file_paths += obj['_filePaths']
+
+        return file_paths
+
+    def __getattr__(self, name):
+        val = getattr(self._wrap, name, None)
+        if val:
+            return val
+        else:
+            raise AttributeError('\'Publication\' has no attribute \'{}\''.format(name))
 
 class CMSIndexed(DocType):
     class Meta:
@@ -307,7 +422,7 @@ def to_dict(self):
         obj_dict = self._doc.to_dict()
         obj_dict['system'] = self.system
         obj_dict['path'] = self.path
-        obj_dict['children'] = [doc.to_dict() for doc in self.children]
+        obj_dict['children'] = [doc.to_dict() if not hasattr(doc, 'projectId') else doc.to_file() for doc in self.children]
         obj_dict['metadata'] = self.metadata()
         obj_dict['permissions'] = 'READ'
         obj_dict['trail'] = self.trail()
@@ -338,6 +453,23 @@ def __init__(self):
     def listing(self, system, file_path, offset=0, limit=100):
         file_path = file_path or '/'
         listing = PublicObject.listing(system, file_path, offset, limit)
+        publications = Publication.listing()
+        #children = [{'agavePath': 'agave://designsafe.storage.published/{}'.format(pub.project.value.projectId),
+        #             'children': [],
+        #             'deleted': False,
+        #             'format': 'folder',
+        #             'length': 24731027,
+        #             'meta': {
+        #                 'title': pub.project['value']['title']
+        #             },
+        #             'name': pub.project.value.projectId,
+        #             'path': '/{}'.format(pub.project.value.projectId),
+        #             'permissions': 'READ',
+        #             'project': pub.project.value.projectId,
+        #             'system': 'designsafe.storage.published',
+        #             'systemId': 'designsafe.storage.published',
+        #             'type': 'dir'} for pub in publications]
+        listing.children = itertools.chain(publications, listing.children)
         return listing
 
     def search(self, system, query_string, 
@@ -442,3 +574,13 @@ def search(self, system, query_string,
             'permissions': 'READ'
         }
         return result
+
+class PublicationManager(object):
+    def save_publication(self, publication):
+        publication['projectId'] = publication['project']['value']['projectId']
+        publication['created'] = datetime.datetime.now().isoformat()
+        publication['status'] = 'publishing'
+        pub = Publication(publication)
+        pub.save()
+        return pub
+