Merge pull request #85 from x-atlas-consortia/simmons/13mar

Changes to field-* endpoints reflect new content from CEDAR, CEDAR_ENTITY

hs-ontology-api-spec.yaml

@@ -2,7 +2,7 @@ openapi: 3.0.3
 info:
   title: HubMAP/SenNet Ontology API (hs-ontology-api)
   description: The HuBMAP/SenNet Ontology API contains endpoints for querying a [UBKG](https://ubkg.docs.xconsortia.org/) instance with content from the [HuBMAP/SenNet context](https://ubkg.docs.xconsortia.org/contexts/#hubmapsennet-context). The hs-ontology-api imports the [ubkg-api](https://smart-api.info/ui/96e5b5c0b0efeef5b93ea98ac2794837), which encapsulates both basic connectivity to a UBKG instance and generic endpoint code.
-  version: 1.4.4
+  version: 1.4.11
   contact:
     name: GitHub repository
     url: https://github.com/x-atlas-consortia/hs-ontology-api
@@ -735,24 +735,33 @@ paths:
   /field-entities:
     get:
       operationId: field_entities_get
-      summary: Return associations between ingest metadata fields and provenance entities. Replacement for field-entities.yaml. NOTE - available only for fields from field-entities.yaml.
+      summary: Return associations between ingest metadata fields and provenance entities. Replacement for field-entities.yaml.
       parameters:
         - name: source
           in: query
           required: false
-          description: case-insensitive name of the ontology source for the provenance entities (HMFIELD = field-types.yaml; HUBMAP = UBKG)
+          description: case-insensitive name of the ontology source for the provenance entity mappings. (HMFIELD = from legacy field-entities.yaml)
           schema:
             type: string
             enum:
               - HMFIELD
-              - HUBMAP
+              - CEDAR
         - name: entity
           in: query
           required: false
-          description: case-sensitive name for entity in either HMFIELD or HUMBMAP ontology
+          description: case-sensitive name for an entity in either HMFIELD or HUBMAP/SENNET
           schema:
             type: string
             example: Sample
+        - name: application_context
+          in: query
+          required: false
+          description: case-insensitive name of the application context
+          schema:
+            type: string
+            enum:
+              - HUBMAP
+              - SENNET
       responses:
         '200':
           description: Associations between ingest metadata fields and provenance entities.
@@ -761,15 +770,15 @@ paths:
               schema:
                 $ref: '#/components/schemas/FieldEntitiesResponse'
         '400':
-          description:  Invalid value for parameter (e.g., *source* not HMFIELD or HUBMAP); invalid parameter
+          description:  Invalid value for parameter (e.g., *source* not HMFIELD or CEDAR); invalid parameter
         '404':
           description: No field entities (with list of parameters, if specified)
         '5XX':
           description: Unexpected error
   /field-entities/{name}:
     get:
       operationId: field_entities_name_get
-      summary: Return associations between specified ingest metadata field and provenance entities. Replacement for field-entities.yaml. NOTE - available only for fields from field-entities.yaml.
+      summary: Return associations between specified ingest metadata field and provenance entities. Replacement for field-entities.yaml.
       parameters:
         - name: name
           in: path
@@ -781,7 +790,7 @@ paths:
         - name: source
           in: query
           required: false
-          description: case-insensitive name of the ontology source for the provenance entities (HMFIELD = field-types.yaml; HUBMAP = UBKG)
+          description: case-insensitive name of the ontology source for the provenance entity mappings. (HMFIELD = from legacy field-entities.yaml)
           schema:
             type: string
             enum:
@@ -790,10 +799,19 @@ paths:
         - name: entity
           in: query
           required: false
-          description: case-sensitive name for entity in either HMFIELD or HUMBMAP ontology
+          description: case-sensitive name for entity in either HMFIELD or HUBMAP/SENNET
           schema:
             type: string
             example: Sample
+        - name: application_context
+          in: query
+          required: false
+          description: case-insensitive name of the application context
+          schema:
+            type: string
+            enum:
+              - HUBMAP
+              - SENNET
       responses:
         '200':
           description: Associations between specified ingest metadata field and provenance entities.
@@ -810,7 +828,7 @@ paths:
   /field-assays:
     get:
       operationId: field_assays_get
-      summary: Return associations between ingest metadata fields and the "assays" (dataset data types). Replacement for field-assays.yaml. NOTE only those CEDAR fields that are also in legacy field-assays.yaml can be mapped to assays.
+      summary: Return associations between ingest metadata fields and the "assays" (dataset data types). Replacement for field-assays.yaml. NOTE only those CEDAR fields that are also in legacy field-assays.yaml can be mapped to assays. In addition, the response from this endpoint is reliable only for datasets that existed prior to the deployment in 2024 of the assay classifier (aka Rules Engine, aka "soft assay types").
       parameters:
         - name: assay_identifier
           in: query
@@ -849,7 +867,7 @@ paths:
   /field-assays/{name}:
     get:
       operationId: field_assays_name_get
-      summary: Return associations between the specified ingest metadata field and the "assays" (dataset data types). Replacement for field-assays.yaml.NOTE only those CEDAR fields that are also in legacy field-assays.yaml can be mapped to assays.
+      summary: Return associations between the specified ingest metadata field and the "assays" (dataset data types). Replacement for field-assays.yaml. NOTE only those CEDAR fields that are also in legacy field-assays.yaml can be mapped to assays.
       parameters:
         - name: name
           in: path

src/hs_ontology_api/cypher/fieldassays.cypher

@@ -1,6 +1,9 @@
-// Obtains associations between ingest metadat fields and assay dataset types, both for legacy (HMFIELD) and CEDAR.
+// Obtains associations between ingest metadata fields and assay dataset types, both for legacy (HMFIELD) and CEDAR.
 // Used by the field-assays endpoint.
 
+// NOTE: With the deployment of the assay classifier (Rules Engine, or "soft assay types"), the UBKG is no longer the
+// source of truth for assay type. This endpoint is primarily for legacy datasets.
+
 // Identify all metadata fields, from both:
 // - legacy sources (the field_*.yaml files in ingest-validation-tools, and modeled in HMFIELD), child codes of HMFIELD:1000
 // - current sources (CEDAR tempates, modeled in CEDAR), child codes of CEDAR:TemplateField
@@ -64,13 +67,14 @@ CALL
     WITH CUIHMDataset
     OPTIONAL MATCH (pAssay:Concept)-[:has_data_type]->(pDataType:Concept)-[:CODE]->(cDataType:Code)-[r:PT]->(tDataType:Term)
     WHERE pAssay.CUI=CUIHMDataset
-    AND cDataType.SAB='HUBMAP'
+    AND cDataType.SAB ='HUBMAP'
     AND r.CUI=pDataType.CUI
     RETURN CASE WHEN tDataType.name IS NULL THEN 'none' ELSE tDataType.name END AS data_type
 }
 
 // For each HuBMAP Dataset, obtain the "soft assay" dataset type.
-// The "soft assay" dataset type is a member of the Soft Assay Dataset Type hierarchy in HUBMAP, with parent code HUBMAP:C003041.
+// The "soft assay" dataset type is a member of the Soft Assay Dataset Type hierarchy in HUBMAP, with parent code
+// HUBMAP:C003041
 CALL
 {
     WITH CUIHMDataset

src/hs_ontology_api/cypher/fieldentities.cypher

@@ -13,11 +13,13 @@
 /// Identify all metadata fields, from both:
 // - legacy sources (the field_*.yaml files in ingest-validation-tools, and modeled in HMFIELD), child codes of HMFIELD:1000
 // - current sources (CEDAR tempates, modeled in CEDAR), child codes of CEDAR:TemplateField
-// Fields that are in the intersection of HMFIELD and CEDAR share CUIs.
 
 // Collect the HMFIELD and CEDAR codes for each metadata field to flatten to level of field name.
 
-// The field_entities_get_logic in neo4j_logic will replace the field_filter and source_filter variables.
+// The field_entities_get_logic in neo4j_logic will replace variables that start with the dollar sign.
+
+// source_filter allows filtering by mapping source (HMFIELD or CEDAR).
+// field_filter allows filtering by field name.
 
 WITH $source_filter AS source_filter
 CALL
@@ -30,24 +32,49 @@ CALL
      apoc.text.join(COLLECT(DISTINCT cField.CodeID),'|') AS code_ids,
      pField.CUI AS CUIField
 }
-// For each field, get associated entities from HMFIELD and HUBMAP ontologies.
-// Each HMFIELD entity node is cross-referenced to a HUBMAP entity node.
-// (CEDAR fields are currently not associated with entities.)
-// The field_entities_get_logic in neo4j_logic will replace the entity_filter and source_filter variables.
+// For each field, get associated provenance entities.
+// entity_filter allows filtering for provenance entity by name--e.g., "dataset", "Dataset".
+// application_filter allows filtering on application context--i.e., "HUBMAP" or "SENNET".
+
 CALL
 {
-     WITH CUIField, source_filter
-     OPTIONAL MATCH (pField:Concept)-[:used_in_entity]->(pEntity:Concept)-[:CODE]->(cHMFIELDEntity:Code)-[rHMFIELD:PT]->(tHMFIELDEntity:Term),
-     (pEntity:Concept)-[:CODE]->(cHUBMAPEntity:Code)-[rHUBMAP:PT]->(tHUBMAPEntity:Term)
-     WHERE pField.CUI=CUIField AND cHMFIELDEntity.SAB ='HMFIELD' AND rHMFIELD.CUI=pEntity.CUI
-     AND cHUBMAPEntity.SAB='HUBMAP' AND rHUBMAP.CUI=pEntity.CUI
-     $entity_filter
-     RETURN apoc.text.join([CASE WHEN source_filter in ['HMFIELD',''] THEN REPLACE(cHMFIELDEntity.CodeID,':','|') + '|' + tHMFIELDEntity.name ELSE '' END,
-     CASE WHEN source_filter in ['HUBMAP',''] THEN REPLACE(cHUBMAPEntity.CodeID,':','|') + '|' + tHUBMAPEntity.name ELSE '' END],';') AS entity
+    // Each HMFIELD field node is linked to a HMFIELD entity node.
+    WITH CUIField,source_filter
+    OPTIONAL MATCH (pField:Concept)-[:used_in_entity]->(pEntity:Concept)-[:CODE]->(cEntity:Code)-[r:PT]->(tEntity:Term)
+    WHERE pField.CUI=CUIField
+    AND cEntity.SAB ='HMFIELD'
+    AND r.CUI=pEntity.CUI
+    $entity_filter
+    RETURN DISTINCT CASE WHEN source_filter IN ['HMFIELD',''] THEN REPLACE(cEntity.CodeID,':','|') + '|' + tEntity.name ELSE '' END AS entity
+
+    UNION
+
+    // Each HMFIELD entity node is cross-referenced to HUBMAP and SENNET provenance entity nodes.
+    WITH CUIField,source_filter
+    OPTIONAL MATCH (pField:Concept)-[:used_in_entity]->(pEntity:Concept)-[:CODE]->(cEntity:Code)-[r:PT]->(tEntity:Term)
+    WHERE pField.CUI=CUIField
+    $application_filter
+    AND r.CUI=pEntity.CUI
+    $entity_filter
+    RETURN DISTINCT CASE WHEN source_filter IN ['HMFIELD',''] THEN REPLACE(cEntity.CodeID,':','|') + '|' + tEntity.name ELSE '' END AS entity
+
+    UNION
+
+    //CEDAR template nodes are mapped to provenance entity nodes in both HUBMAP and SENNET.
+    //CEDAR field nodes relate to CEDAR template nodes.
 
+    WITH CUIField,source_filter
+    OPTIONAL MATCH (pField:Concept)-[:inverse_has_field]->(pTemplate:Concept)-[:used_in_entity]->(pEntity:Concept)-[:CODE]->(cEntity:Code)-[r:PT]->(tEntity:Term)
+    WHERE pField.CUI = CUIField
+    AND r.CUI=pEntity.CUI
+    //AND c.Entity.SAB in ['HUBMAP','SENNET']
+    $application_filter
+    $entity_filter
+    RETURN DISTINCT CASE WHEN source_filter IN ['CEDAR',''] THEN REPLACE(cEntity.CodeID,':','|') + '|' + tEntity.name ELSE '' END AS entity
 }
 
+WITH field_name, code_ids, entity
+WHERE entity <>""
 WITH field_name, code_ids, COLLECT(entity) AS entities
-WHERE entities <>['null;null']
 RETURN field_name, code_ids, entities
 ORDER BY field_name

src/hs_ontology_api/routes/fieldentities/fieldentities_controller.py

@@ -12,31 +12,40 @@
 def field_entities_get(name=None):
     """Returns detailed information on field entities.
 
+    MARCH 2024 - updated for new CEDAR content and CEDAR_ENTITY ontology.
+
     :rtype: Union[List[FieldEntity]]
 
     """
 
     # Validate parameters
-    err = validate_query_parameter_names(['source', 'entity'])
+    err = validate_query_parameter_names(['source', 'entity','application_context'])
     if err != 'ok':
         return make_response(err, 400)
 
-
     # Validate mapping source.
     source = request.args.get('source')
     if source is not None:
         source = source.upper()
-        val_enum = ['HMFIELD', 'HUBMAP']
+        val_enum = ['HMFIELD', 'CEDAR']
         err = validate_parameter_value_in_enum(param_name='source', param_value=source, enum_list=val_enum)
         if err != 'ok':
             return make_response(err, 400)
 
+    # Validate application context.
+    application = request.args.get('application_context')
+    if application is not None:
+        application = application.upper()
+        val_enum = ['HUBMAP', 'SENNET']
+        err = validate_parameter_value_in_enum(param_name='application_context', param_value=application, enum_list=val_enum)
+        if err != 'ok':
+            return make_response(err, 400)
 
     entity = request.args.get('entity')
 
     neo4j_instance = current_app.neo4jConnectionHelper.instance()
     result = field_entities_get_logic(neo4j_instance, field_name=name, source=source,
-                                   entity=entity)
+                                   entity=entity, application=application)
     if result is None or result == []:
         # Empty result
         err = get_404_error_string(prompt_string='No field type associations')