Merge pull request #133 from x-atlas-consortia/jas_organ_category

Jas organ category

hs-ontology-api-spec.yaml

@@ -129,6 +129,15 @@ paths:
               - sennet
               - HuBMAP
               - SenNet
+        - name: is_externally_processed
+          in: query
+          required: false
+          description: filter on whether dataset types are for EPIC datasets
+          schema:
+            type: string
+            enum:
+              - true
+              - false
       responses:
         '200':
           description: A JSON array of dataset type objects
@@ -170,6 +179,15 @@ paths:
               - sennet
               - HuBMAP
               - SenNet
+        - name: is_externally_processed
+          in: query
+          required: false
+          description: filter on whether dataset types are for EPIC datasets
+          schema:
+            type: string
+            enum:
+              - true
+              - false
       responses:
         '200':
           description: A dataset type object
@@ -1078,6 +1096,22 @@ components:
         organ_cui:
           type: string
           description: The organ CUI
+        laterality:
+          type: string
+          description: laterality of the organ-- left, right, or null
+          example: left
+        category:
+          type: object
+          description: the organ's category. Used primarily for organs that exhibit laterality.
+          properties:
+            organ_uberon:
+              type: string
+              description: UBERON code
+              example: "UBERON:0002185"
+            term:
+              type: string
+              description: name of the organ category
+              example: Bronchus
     HGNCIdRelationships:
       type: object
       description: Response Body for relationships/hgnc-id/{id} GET request
@@ -1935,10 +1969,6 @@ components:
                   type: string
                   description: whether datasets generated by the processing workflow associated with the assay class are "primary" (from the original experiment); "derived"/"processed" (from post-experimental data processing); or "epic" (Externally Processed Integrated Collections)
                   example: primary
-                provider:
-                  type: string
-                  description: the provider of datasets generated by the processing workflow associate with the assay class. Possible values are "IEC" (corresponding to datasets generated by the HuBMAP/SenNet Infrastructure and Engagement Component) or "External" (complete datasets coming directly from a HuBMAP/SenNet lab--e.g., a Tissue Mapping Center).
-                  example: IEC
                 vitessce_hints:
                   type: array
                   description: set of keys used to identify the type of visualization that the Vitessce application should use to represent datasets associated with the assay class.
@@ -1983,3 +2013,7 @@ components:
                   items:
                     type: string
                     example: IMC2D
+                is_externally_processed:
+                  type: string
+                  description: whether the dataset type is for an EPIC dataset
+                  example: false

src/hs_ontology_api/cypher/assayclass.cypher

@@ -164,14 +164,7 @@ CALL
         AND ppii.CUI = context+':C004009 CUI'
         RETURN DISTINCT CASE WHEN NOT ppii.CUI IS null THEN true ELSE false END AS contains_full_genetic_sequences
 }
-// provider
-CALL
-{
-        WITH CUIRBD,context
-        OPTIONAL MATCH (pRBD:Concept)-[:has_provider]->(pProvider:Concept)-[:CODE]-(cProvider:Code)-[r:PT]->(tProvider:Term)
-        WHERE pRBD.CUI=CUIRBD AND r.CUI=pProvider.CUI AND cProvider.SAB=context
-        RETURN DISTINCT tProvider.name AS provider
-}
+
 // active status
 CALL
 {
@@ -182,7 +175,12 @@ CALL
 }
 CALL
 {
-WITH context, CodeRBD, NameRBD, assaytype, dir_schema, tbl_schema, vitessce_hints,process_state,pipeline_shorthand,description,dataset_type,pdr_category,fig2_aggregated_assaytype,fig2_modality,fig2_category,is_multiassay,must_contain,MeasCodes,contains_full_genetic_sequences,provider,active_status
+WITH
+context, CodeRBD, NameRBD, assaytype, dir_schema, tbl_schema,
+vitessce_hints,process_state,pipeline_shorthand,
+description,dataset_type,pdr_category,
+fig2_aggregated_assaytype,fig2_modality,fig2_category,
+is_multiassay,must_contain,MeasCodes,contains_full_genetic_sequences,active_status
 RETURN
 {
         rule_description:
@@ -194,7 +192,6 @@ RETURN
                 process_state:process_state,
                 pipeline_shorthand:pipeline_shorthand, description:description,
                 is_multiassay:is_multiassay, must_contain:must_contain,
-                provider:provider,
                 active_status:active_status,
                 dataset_type:
                 {

src/hs_ontology_api/cypher/datasettypes.cypher

@@ -64,7 +64,18 @@ CALL
         AND cAssayType.SAB=context
         RETURN COLLECT(DISTINCT tAssayType.name) AS assaytypes
 }
-WITH  dataset_type,pdr_category,fig2_aggregated_assaytype,fig2_modality,fig2_category,assaytypes
+// Whether an Epic datatype
+CALL
+{
+   WITH CUIDatasetType,context
+   OPTIONAL MATCH (pDatasetType:Concept)-[:isa]->(pEpic:Concept)-[:CODE]->(cEpic:Code)
+   WHERE pDatasetType.CUI = CUIDatasetType
+   AND cEpic.CODE = 'C004034'
+   AND cEpic.SAB = context
+   RETURN DISTINCT CASE WHEN pEpic IS NULL THEN false ELSE true END AS is_externally_processed
+}
+WITH  dataset_type,pdr_category,fig2_aggregated_assaytype,fig2_modality,fig2_category,assaytypes,is_externally_processed
+$epictype_filter
 RETURN
 {
         dataset_type:dataset_type,
@@ -75,5 +86,6 @@ RETURN
             modality:fig2_modality,
             category:fig2_category
         },
-        assaytypes:assaytypes
+        assaytypes:assaytypes,
+        is_externally_processed:is_externally_processed
 } AS dataset_types

src/hs_ontology_api/cypher/organs.cypher

@@ -1,3 +1,5 @@
+// JAS SEPT 2024 - Converted return to JSON. Added organ category and laterality.
+
 //  Replaces and extends a read of the organ_types.yaml in the search-api. Used by endpoints in the organs route.
 
 // The calling function in neo4j_logic.py will replace $sab.
@@ -30,7 +32,47 @@ CALL
 	WITH OrganCUI OPTIONAL MATCH (pOrgan:Concept)-[r1:has_two_character_code]->(p2CC:Concept)-[r2:PREF_TERM]->(t2CC:Term)
 	WHERE pOrgan.CUI=OrganCUI AND r1.SAB=$sab RETURN t2CC.name as OrganTwoCharacterCode
 }
+// Organ categories
+CALL
+{
+   WITH OrganCUI
+   OPTIONAL MATCH (pOrgan:Concept)-[:isa]-(pOrganCat:Concept)-[:isa]->(pCat:Concept)-[:CODE]->(cCat:Code),
+   // HuBMAP name for the category
+   (pOrganCat:Concept)-[:CODE]->(cOrganCat:Code)-[rOrganCat:PT]-(tOrganCat:Term),
+   // UBERON code for the category
+   (pOrganCat:Concept)-[:CODE]->(cUBERON:Code)
+   WHERE pOrgan.CUI = OrganCUI
+   //Organ cat parent
+   AND cCat.SAB=$sab
+   AND cCat.CODE='C045000'
+   AND cOrganCat.SAB=$sab
+   AND rOrganCat.CUI=pOrganCat.CUI
+   AND cUBERON.SAB='UBERON'
+   RETURN DISTINCT
+   CASE
+   // Kidney mapped to both kidney and mammalian kidney
+   WHEN OrganCUI in ['C0227614','C0227613'] THEN 'UBERON:0002113'
+   // Lung mapped to both lung and pair of lungs
+   WHEN OrganCUI in ['C0225730','C0225706'] THEN 'UBERON:0002048'
+   ELSE cUBERON.CodeID END AS OrganCatUBERON,tOrganCat.name AS OrganCatTerm
+}
+// Laterality
+CALL
+{
+    WITH OrganCUI
+    OPTIONAL MATCH (pOrgan:Concept)-[:has_laterality]->(pLaterality:Concept)-[:CODE]->(cLaterality:Code)-[rLaterality:PT]->(tLaterality:Term)
+    WHERE pOrgan.CUI = OrganCUI
+    AND rLaterality.CUI = pLaterality.CUI
+    AND cLaterality.SAB=$sab
+    // Return null for 'No Laterality' or 'Unknown Laterality'
+    RETURN DISTINCT CASE WHEN cLaterality.CODE IN ['C030039','C030040','C030041','C030022','C030023'] THEN NULL ELSE REPLACE(tLaterality.name," Laterality","") END AS laterality
+}
 // Filter out the "Other" organ node.
-WITH OrganCode,OrganSAB,OrganName,OrganTwoCharacterCode,OrganUBERON,OrganFMA,OrganCUI
+WITH OrganCode,OrganSAB,OrganName,OrganTwoCharacterCode,OrganUBERON,OrganFMA,OrganCUI,laterality,
+CASE WHEN OrganCatUBERON IS NULL THEN NULL ELSE {organ_uberon:OrganCatUBERON, term:OrganCatTerm} END AS category
+
 WHERE NOT (OrganCode = 'C030071' AND OrganSAB=$sab)
-RETURN DISTINCT OrganCode,OrganSAB,OrganName,CASE WHEN OrganUBERON IS NULL THEN OrganFMA ELSE OrganUBERON END AS OrganUBERON,OrganTwoCharacterCode,OrganCUI ORDER BY OrganName
+RETURN DISTINCT {code:OrganCode, sab:OrganSAB, term:OrganName,
+organ_uberon:CASE WHEN OrganUBERON IS NULL THEN OrganFMA ELSE OrganUBERON END,
+rui_code:OrganTwoCharacterCode, organ_cui:OrganCUI, laterality:laterality, category:category} AS organ
+ORDER BY organ.term

src/hs_ontology_api/routes/datasettypes/datasettypes_controller.py

@@ -19,11 +19,13 @@ def datasettypes_get(name=None):
     in the testing rules json, with options to filter the list to those with specific property values.
     Filters are additive (i.e., boolean AND)
 
+    JAS Sept 2024 - added isepic filter
+
     """
     # Validate parameters.
 
     # Check for invalid parameter names.
-    err = validate_query_parameter_names(parameter_name_list=['application_context'])
+    err = validate_query_parameter_names(parameter_name_list=['application_context','is_externally_processed'])
     if err != 'ok':
         return make_response(err, 400)
 
@@ -40,9 +42,21 @@ def datasettypes_get(name=None):
         return make_response(err, 400)
     application_context = application_context.upper()
 
+    # Check for valid isepic. The parameter is case-insensitive
+    val_enum = ['TRUE','FALSE']
+    isepic = request.args.get('is_externally_processed')
+    if isepic is not None:
+        err = validate_parameter_value_in_enum(param_name='is_externally_processed', param_value=isepic.upper(),
+                                           enum_list=val_enum)
+        if err != 'ok':
+            return make_response(err, 400)
+
+    if isepic is not None:
+        isepic = isepic.lower()
+
     neo4j_instance = current_app.neo4jConnectionHelper.instance()
     result = datasettypes_get_logic(
-            neo4j_instance, datasettype=name, context=application_context)
+            neo4j_instance, datasettype=name, context=application_context, isepic=isepic)
 
     if result is None or result == []:
         # Empty result

src/hs_ontology_api/utils/neo4j_logic.py

@@ -211,6 +211,8 @@ def get_organ_types_logic(neo4j_instance, sab):
     :param neo4j_instance: pointer to neo4j connection
     :return:
 
+    JAS SEPT 2024 - Converted to using JSON returned from query. Added organ category and laterality.
+
     JAS NOV 2023 - Moved query string to external file and implemented loadquery utility logic.
     """
     result = []
@@ -223,14 +225,16 @@ def get_organ_types_logic(neo4j_instance, sab):
 
     with neo4j_instance.driver.session() as session:
         recds: neo4j.Result = session.run(query)
+        #for record in recds:
+            #item = SabCodeTermRuiCode(sab=record.get('OrganSAB'), code=record.get('OrganCode'),
+                                      #term=record.get('OrganName'), rui_code=record.get('OrganTwoCharacterCode'),
+                                      #organ_uberon=record.get('OrganUBERON'), organ_cui=record.get('OrganCUI')
+                                      #).serialize()
+            #result.append(item)
         for record in recds:
-            item = SabCodeTermRuiCode(sab=record.get('OrganSAB'), code=record.get('OrganCode'),
-                                      term=record.get('OrganName'), rui_code=record.get('OrganTwoCharacterCode'),
-                                      organ_uberon=record.get('OrganUBERON'), organ_cui=record.get('OrganCUI')
-                                      ).serialize()
-            result.append(item)
-    return result
+            result.append(record.get('organ'))
 
+        return result
 
 def relationships_for_gene_target_symbol_get_logic(neo4j_instance, target_symbol: str) -> dict:
     """
@@ -1486,7 +1490,7 @@ def assayclasses_get_logic(neo4j_instance,assayclass=None, assaytype=None, proce
 
     return assayclasses
 
-def datasettypes_get_logic(neo4j_instance,datasettype=None, context=None) -> dict:
+def datasettypes_get_logic(neo4j_instance,datasettype=None, context=None, isepic=None) -> dict:
     """
     July 2024
         Obtains information on dataset types.
@@ -1496,6 +1500,7 @@ def datasettypes_get_logic(neo4j_instance,datasettype=None, context=None) -> dic
         :param neo4j_instance: neo4j connection
         :param datasettype: dataset_type
         :param context: application context--i.e., HUBMAP or SENNET
+        :param isepic: optional filter to Epic (externally processed) dataset types
 
         """
     datasettypes: [dict] = []
@@ -1512,7 +1517,15 @@ def datasettypes_get_logic(neo4j_instance,datasettype=None, context=None) -> dic
     else:
         querytxt = querytxt.replace('$datasettype_filter','')
 
-    print(querytxt)
+    if isepic in ['true','false']:
+        if isepic == 'true':
+            isepicbool = True
+        else:
+            isepicbool = False
+        querytxt = querytxt.replace('$epictype_filter', f"WHERE is_externally_processed={isepicbool}")
+    else:
+        querytxt = querytxt.replace('$epictype_filter','')
+
     # Set timeout for query based on value in app.cfg.
     query = neo4j.Query(text=querytxt, timeout=neo4j_instance.timeout)
 

test/test_api.sh

@@ -186,11 +186,21 @@ echo
 echo | tee -a test.out
 echo | tee -a test.out
 
-echo "3. /dataset-types?application_context=HUBMAP => valid; should return 200" | tee -a test.out
+echo "3. /dataset-types?application_context=HUBMAP&is_externally_processed=mango => invalid parameter; should return custom 400" | tee -a test.out
+curl --request GET \
+ --url "${UBKG_URL}/dataset-types?application_context=HUBMAP&is_externally_processed=mango" \
+ --header "Accept: application/json" | cut -c1-60 | tee -a test.out
+echo
+echo "4. /dataset-types?application_context=HUBMAP => valid; should return 200" | tee -a test.out
 curl --request GET \
  --url "${UBKG_URL}/dataset-types?application_context=HUBMAP" \
  --header "Accept: application/json" | cut -c1-60 | tee -a test.out
 echo
+echo "5. /dataset-types?application_context=HUBMAP&is_externally_processed=false => valid; should return 200" | tee -a test.out
+curl --request GET \
+ --url "${UBKG_URL}/dataset-types?application_context=HUBMAP&is_externally_processed=true" \
+ --header "Accept: application/json" | tee -a test.out
+
 
 # dataset-types/<id> uses the same code as dataset-types
 echo "TESTS FOR: dataset-types/<id> GET" | tee -a test.out

test/test_gateway.sh

@@ -140,6 +140,16 @@ curl --request GET \
  --header "Accept: application/json" |cut -c1-60
 echo
 
+echo "dataset-types GET"
+curl --request GET \
+ --url "${UBKG_URL}/dataset-types?application_context=HUBMAP" \
+ --header "Accept: application/json" |cut -c1-60
+
+echo "dataset-types/<name> GET"
+curl --request GET \
+ --url "${UBKG_URL}/dataset-types/Auto-fluorescence?application_context=HUBMAP" \
+ --header "Accept: application/json" |cut -c1-60
+
 echo "organs GET for HUBMAP"
 curl --request GET \
  --url "${UBKG_URL}/organs?application_context=HUBMAP" \