Merge pull request #414 from hubmapconsortium/test-release

v2.2.1 release

VERSION

@@ -1 +1 @@
-2.2.0
+2.2.1

search-api-spec.yaml

@@ -6,7 +6,48 @@ info:
   title: HuBMAP Search API
   termsOfService: 'https://hubmapconsortium.org'
   contact:
-    email: [email protected]
+    name: HuBMAP Help Desk
+    email: [email protected]
+  license:
+    name: MIT License
+    url: 'https://github.com/hubmapconsortium/entity-api/blob/master/license.txt'
+servers:
+  - url: "https://search.api.hubmapconsortium.org"
+components:
+  securitySchemes:
+    globus_auth:
+      type: oauth2
+      flows:
+        implicit:
+          authorizationUrl: "https://auth.globus.org/v2/oauth2/authorize"
+          scopes:
+            "write:entities": modify entities
+            "read:entities": read entities
+    JWTBearerAuth:
+      bearerFormat: JWT
+      scheme: bearer
+      type: http
+  schemas:
+    requestJsonBody:
+      type: object
+    AssayTypeQueryDict:
+      type: object
+      properties:
+        name:
+          type: string
+    AssayTypeReturnDict:
+      type: object
+      properties:
+        name:
+          type: string
+        description:
+          type: string
+        primary:
+          type: boolean
+        vitessce-hints:
+          type: array
+          items:
+            type: string
 tags:
   - name: 'Search API, Elasticsearch'
     description: Operations pertaining to datasets indexed in Elasticsearch
@@ -268,40 +309,4 @@ paths:
         '202':
           description: The request has been accepted and reindex is in process
 
-servers:
-  - url: "https://search.api.hubmapconsortium.org"
-components:
-  securitySchemes:
-    globus_auth:
-      type: oauth2
-      flows:
-        implicit:
-          authorizationUrl: "https://auth.globus.org/v2/oauth2/authorize"
-          scopes:
-            "write:entities": modify entities
-            "read:entities": read entities
-    JWTBearerAuth:
-      bearerFormat: JWT
-      scheme: bearer
-      type: http
-  schemas:
-    requestJsonBody:
-      type: object
-    AssayTypeQueryDict:
-      type: object
-      properties:
-        name:
-          type: string
-    AssayTypeReturnDict:
-      type: object
-      properties:
-        name:
-          type: string
-        description:
-          type: string
-        primary:
-          type: boolean
-        vitessce-hints:
-          type: array
-            items:
-              type: string
+

src/elasticsearch/addl_index_transformations/portal/__init__.py

@@ -56,6 +56,7 @@ def transform(doc, batch_id='unspecified'):
     >>> transformed = transform({
     ...    'entity_type': 'dataset',
     ...    'status': 'New',
+    ...    'group_name': 'EXT - Outside HuBMAP',
     ...    'origin_sample': {
     ...        'organ': 'LY'
     ...    },
@@ -87,7 +88,8 @@ def transform(doc, batch_id='unspecified'):
     ...            'metadata_path': 'No!',
     ...            'tissue_id': 'No!',
     ...            'donor_id': 'No!',
-    ...            'keep_this_field': 'Yes!'
+    ...            'keep_this_field': 'Yes!',
+    ...            'is_boolean': '1'
     ...        }
     ...    },
     ...    'rui_location': '{"ccf_annotations": ["http://purl.obolibrary.org/obo/UBERON_0001157"]}'
@@ -112,12 +114,14 @@ def transform(doc, batch_id='unspecified'):
                                                   'grouping_concept_preferred_term': 'Sex',
                                                   'preferred_term': 'Male'}]}},
      'entity_type': 'dataset',
+     'group_name': 'EXT - Outside HuBMAP',
      'mapped_create_timestamp': '2019-12-04 19:58:29',
      'mapped_data_access_level': 'Consortium',
      'mapped_data_types': ['snRNA-seq [Salmon]'],
+     'mapped_external_group_name': 'Outside HuBMAP',
      'mapped_metadata': {},
      'mapped_status': 'New',
-     'metadata': {'metadata': {'keep_this_field': 'Yes!'}},
+     'metadata': {'metadata': {'is_boolean': 'TRUE', 'keep_this_field': 'Yes!'}},
      'origin_sample': {'mapped_organ': 'Lymph Node', 'organ': 'LY'},
      'rui_location': '{"ccf_annotations": '
                      '["http://purl.obolibrary.org/obo/UBERON_0001157"]}',
@@ -179,24 +183,44 @@ def _map(doc, clean):
 
 
 def _simple_clean(doc):
-    field = 'created_by_user_displayname'
-    if field in doc and doc[field] == 'daniel Cotter':
-        doc[field] = 'Daniel Cotter'
-    if field in doc and doc[field] == 'amir Bahmani':
-        doc[field] = 'Amir Bahmani'
+    # We shouldn't get messy data in the first place...
+    # but it's just not feasible to make the fixes upstream.
+    if not isinstance(doc, dict):
+        return
+
+    # Clean up names conservatively,
+    # based only on the problems we actually see:
+    name_field = 'created_by_user_displayname'
+    if doc.get(name_field, '').lower() in [
+            'daniel cotter', 'amir bahmani', 'adam kagel', 'gloria pryhuber']:
+        doc[name_field] = doc[name_field].title()
 
+    # Clean up metadata:
     if 'metadata' in doc and 'metadata' in doc['metadata']:
+        metadata = doc['metadata']['metadata']
+
         bad_fields = [
             'collectiontype',  # Inserted by IEC.
-            'data_path', 'metadata_path',  # Only meaningful at submission time.
+            'data_path', 'metadata_path', 'version',  # Only meaningful at submission time.
             'donor_id', 'tissue_id'  # For internal use only.
         ]
-        metadata = {
-            k: v for k, v
-            in doc['metadata']['metadata'].items()
-            if k not in bad_fields and not k.startswith('_')
-        }
-        doc['metadata']['metadata'] = metadata
+
+        # Explicitly convert items to list,
+        # so we can remove keys from the metadata dict:
+        for k, v in list(metadata.items()):
+            if k in bad_fields or k.startswith('_'):
+                del metadata[k]
+            # Normalize booleans to all-caps, the Excel default.
+            # (There is no guaratee that boolean fields with be prefixed this way,
+            # but at the moment it is the case.)
+            if k.startswith('is_'):
+                if v in ['0', 'false', 'False']:
+                    metadata[k] = 'FALSE'
+                if v in ['1', 'true', 'True']:
+                    metadata[k] = 'TRUE'
+        # Other converstions are handled by ES numeric detection.
+        # See: portal/config.yaml
+        # https://www.elastic.co/guide/en/elasticsearch/reference/current/dynamic-field-mapping.html
 
 # TODO: Reenable this when we have time, and can make sure we don't need these fields.
 #

src/elasticsearch/addl_index_transformations/portal/translate.py

@@ -25,6 +25,7 @@ def translate(doc):
     _translate_data_type(doc)
     _translate_timestamp(doc)
     _translate_access_level(doc)
+    _translate_external_consortium(doc)
 
 
 # Utils:
@@ -95,6 +96,25 @@ def _access_level_map(access_level):
     return access_level.title()
 
 
+# External consortium:
+
+def _translate_external_consortium(doc):
+    '''
+    >>> doc = {'group_name': 'Inside HuBMAP'}
+    >>> _translate_external_consortium(doc); doc
+    {'group_name': 'Inside HuBMAP'}
+    >>> doc = {'group_name': 'EXT - Outside HuBMAP'}
+    >>> _translate_external_consortium(doc); doc
+    {'group_name': 'EXT - Outside HuBMAP', 'mapped_external_group_name': 'Outside HuBMAP'}
+
+    '''
+    group_name = doc.get('group_name')
+    if group_name is None:
+        return
+    if 'EXT' in group_name:
+        doc['mapped_external_group_name'] = group_name.replace('EXT - ', '')
+
+
 # Timestamp:
 
 def _translate_timestamp(doc):

src/search-schema/data/definitions/enums/assay_types.yaml

@@ -40,7 +40,7 @@ celldive_deepcell:
     alt-names: []
     primary: false
     contains-pii: false
-    vitessce-hints: ['codex', 'is_image', 'is_tiled']
+    vitessce-hints: ['sprm', 'anndata', 'is_image', 'is_tiled']
 
 CODEX:
     description: CODEX
@@ -159,7 +159,7 @@ MALDI-IMS:
     alt-names: ['MALDI-IMS-neg', 'MALDI-IMS-pos']
     primary: true
     contains-pii: false
-    vitessce-hints: []
+    vitessce-hints: ['maldi']
 
 MALDI-IMS_pyramid:
     description: MALDI IMS [Image Pyramid]
@@ -173,46 +173,38 @@ MALDI-IMS_pyramid:
     primary: false
     vis-only: true
     contains-pii: false
-    vitessce-hints: ['pyramid']
-
-#
-# The following block of definitions is in the process of being replaced by the
-# 'MALDI-IMS' and 'MALDI-IMS_pyramid' blocks above, but the original version is
-# being preserved here in comments pending full testing of the replacement.
-#
-# MALDI-IMS-neg:
-#     description: MALDI IMS negative
-#     alt-names: []
-#     primary: true
-#     contains-pii: false
-#     vitessce-hints: []
-#
-# MALDI-IMS-neg_pyramid:
-#     description: MALDI IMS negative [Image Pyramid]
-#     alt-names: [["MALDI-IMS-neg", "Image Pyramid"], ["MALDI-IMS-neg", "image_pyramid"]]
-#     primary: false
-#     vis-only: true
-#     contains-pii: false
-#     vitessce-hints: ['pyramid']
-#
-# MALDI-IMS-pos:
-#     description: MALDI IMS positive
-#     alt-names: []
-#     primary: true
-#     contains-pii: false
-#     vitessce-hints: []
-#
-# MALDI-IMS-pos_pyramid:
-#     description: MALDI IMS positive [Image Pyramid]
-#     alt-names: [["MALDI-IMS-pos", "Image Pyramid"], ["MALDI-IMS-pos", "image_pyramid"]]
-#     primary: false
-#     vis-only: true
-#     contains-pii: false
-#     vitessce-hints: ['pyramid']
-#
-# End block of earlier MALDI-IMS code being preserved during testing
-#
-
+    vitessce-hints: ['pyramid', 'maldi']
+
+NanoDESI:
+  description: NanoDESI
+  alt-names: []
+  primary: true
+  contains-pii: false
+  vitessce-hints: []
+
+NanoDESI_pyramid:
+  description: NanoDESI [Image Pyramid]
+  alt-names: [["NanoDESI", "Image Pyramid"], ["NanoDESI", "image_pyramid"]]
+  primary: false
+  vis-only: true
+  contains-pii: false
+  vitessce-hints: ['pyramid']
+
+NanoPOTS:
+  description: NanoPOTS
+  alt-names: []
+  primary: true
+  contains-pii: false
+  vitessce-hints: []
+
+NanoPOTS_pyramid:
+  description: NanoPOTS [Image Pyramid]
+  alt-names: [["NanoPOTS", "Image Pyramid"], ["NanoPOTS", "image_pyramid"]]
+  primary: false
+  vis-only: true
+  contains-pii: false
+  vitessce-hints: ['pyramid']
+
 MxIF:
     description: Multiplexed IF Microscopy
     alt-names: []
@@ -262,7 +254,7 @@ salmon_rnaseq_bulk:
 
 SNARE-ATACseq2:
     description: snATACseq (SNARE-seq2)
-    alt-names: ['SNAREseq', 'SNARE2-ATACseq']
+    alt-names: ['SNAREseq', 'SNARE-seq2', 'SNARE2-ATACseq']
     primary: true
     contains-pii: true
     vitessce-hints: []
@@ -450,4 +442,46 @@ WGS:
     contains-pii: true
     vitessce-hints: []
 
-
+LC-MS:
+    description: LC-MS
+    alt-names: []
+    primary: true
+    contains-pii: false
+    vitessce-hints: []
+
+MS:
+    description: MS
+    alt-names: []
+    primary: true
+    contains-pii: false
+    vitessce-hints: []
+
+LC-MS_bottom_up:
+    description: 'LC-MS Bottom Up'
+    alt-names: []
+    primary: true
+    contains-pii: false
+    vitessce-hints: []
+
+MS_bottom_up:
+    description: 'MS Bottom Up'
+    alt-names: []
+    primary: true
+    contains-pii: false
+    vitessce-hints: []
+
+LC-MS_top_down:
+    description: 'LC-MS Top Down'
+    alt-names: []
+    primary: true
+    contains-pii: false
+    vitessce-hints: []
+
+MS_top_down:
+    description: 'MS Top Down'
+    alt-names: []
+    primary: true
+    contains-pii: false
+    vitessce-hints: []
+
+