From 33859ad270a831f75624a3662a32984fbab49507 Mon Sep 17 00:00:00 2001 From: ChuckKollar Date: Fri, 19 Jan 2024 15:28:00 -0500 Subject: [PATCH 01/22] Updated neo4j in requirements.txt --- src/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/requirements.txt b/src/requirements.txt index 2a669e4..220c45c 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,6 +1,6 @@ ubkg-api==1.4.0 Flask == 2.1.3 -neo4j == 4.4 +neo4j == 5.15.0 # for analysis of tabular data pandas==1.5.0 @@ -12,4 +12,4 @@ numpy==1.23.5 Werkzeug==2.3.7 # Cells API client -hubmap-api-py-client==0.0.9 \ No newline at end of file +hubmap-api-py-client==0.0.9 From f78d31847dfa3736e94998792c1e65926293ee66 Mon Sep 17 00:00:00 2001 From: ChuckKollar Date: Fri, 26 Jan 2024 14:04:25 -0500 Subject: [PATCH 02/22] Changed to Neo4J v5, fixed depreceted cyphers, added test information. --- .../cypher/celltypedetail.cypher | 8 +++-- src/hs_ontology_api/cypher/genedetail.cypher | 17 ++++++---- src/requirements.txt | 7 ++-- test_api.sh | 34 ++++++++++++++++++- 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/src/hs_ontology_api/cypher/celltypedetail.cypher b/src/hs_ontology_api/cypher/celltypedetail.cypher index 765528e..d9b5c95 100644 --- a/src/hs_ontology_api/cypher/celltypedetail.cypher +++ b/src/hs_ontology_api/cypher/celltypedetail.cypher @@ -59,8 +59,10 @@ UNION WITH CLCUI OPTIONAL MATCH (cCL:Code)<-[:CODE]-(pCL:Concept)-[:has_marker_component]->(pGene:Concept)-[:CODE]->(cGene:Code)-[r]->(tGene:Term) WHERE pCL.CUI=CLCUI AND cGene.SAB='HGNC' AND r.CUI=pGene.CUI AND cCL.SAB='CL' AND type(r) IN ['ACR','PT'] -RETURN distinct cCL.CodeID as CLID, 'cell_types_genes' as ret_key, cGene.CodeID + '|' + apoc.text.join(COLLECT(tGene.name),'|') AS ret_value -ORDER BY CLID, cGene.CodeID + '|' + apoc.text.join(COLLECT(tGene.name),'|') +WITH COLLECT(tGene.name) AS tgene_names, cGene.CodeID AS cgene_codeid, cCL.CodeID AS ccl_codeid +WITH distinct ccl_codeid AS CLID, 'cell_types_genes' AS ret_key, cgene_codeid+'|'+apoc.text.join(tgene_names,'|') AS ret_value +RETURN CLID, ret_key, ret_value +ORDER BY CLID, ret_value UNION @@ -110,4 +112,4 @@ map['cell_types_definition'] AS cell_types_definition, map['cell_types_genes'] AS cell_types_genes, map['cell_types_organ'] AS cell_types_organs -order by CLID \ No newline at end of file +order by CLID diff --git a/src/hs_ontology_api/cypher/genedetail.cypher b/src/hs_ontology_api/cypher/genedetail.cypher index e25e6f3..315a6b6 100644 --- a/src/hs_ontology_api/cypher/genedetail.cypher +++ b/src/hs_ontology_api/cypher/genedetail.cypher @@ -98,7 +98,8 @@ WITH hgnc_id, CLID, mintype OPTIONAL MATCH (cCL:Code)-[rCL]->(tCL:Term) where cCL.CodeID = CLID AND type(rCL) STARTS WITH 'PT' AND CASE WHEN type(rCL)='PT' THEN 0 ELSE 1 END=mintype -return hgnc_id, 'cell_types_name' AS ret_key, CLID +'|'+ CASE WHEN tCL.name IS NULL THEN '' ELSE tCL.name END AS ret_value +WITH hgnc_id, 'cell_types_name' AS ret_key, CLID +'|'+ CASE WHEN tCL.name IS NULL THEN '' ELSE tCL.name END AS ret_value +RETURN hgnc_id, ret_key, ret_value UNION @@ -106,8 +107,10 @@ UNION // Definitions link to Concepts and multiple CL codes can match to the same concept; however, each CL code has a "preferred" CUI, identified by the CUI property of the relationship of any of the code's linked terms. WITH GeneCUI -OPTIONAL MATCH (cGene:Code)<-[:CODE]-(pGene:Concept)-[:inverse_has_marker_component]->(pCL:Concept)-[:CODE]->(cCL:Code)-[rCL]->(tCL:Term),(pCL:Concept)-[:DEF]->(dCL:Definition) WHERE rCL.CUI=pCL.CUI AND pGene.CUI=GeneCUI AND cGene.SAB='HGNC' AND cCL.SAB='CL' AND dCL.SAB='CL' RETURN DISTINCT toInteger(cGene.CODE) AS hgnc_id,'cell_types_definition' as ret_key, cCL.CodeID + '|'+ dCL.DEF as ret_value -ORDER BY hgnc_id,cCL.CodeID + '|'+ dCL.DEF +OPTIONAL MATCH (cGene:Code)<-[:CODE]-(pGene:Concept)-[:inverse_has_marker_component]->(pCL:Concept)-[:CODE]->(cCL:Code)-[rCL]->(tCL:Term),(pCL:Concept)-[:DEF]->(dCL:Definition) WHERE rCL.CUI=pCL.CUI AND pGene.CUI=GeneCUI AND cGene.SAB='HGNC' AND cCL.SAB='CL' AND dCL.SAB='CL' +WITH toInteger(cGene.CODE) AS hgnc_id,'cell_types_definition' as ret_key, cCL.CodeID + '|'+ dCL.DEF as ret_value +RETURN DISTINCT hgnc_id, ret_key, ret_value +ORDER BY hgnc_id, ret_value UNION @@ -129,9 +132,9 @@ CALL OPTIONAL MATCH (cAZ:Code)<-[:CODE]-(pAZ:Concept)-[rAZUB:located_in]->(pUB:Concept)-[:CODE]->(cUB:Code)-[rUB:PT]->(tUB:Term) WHERE rAZUB.SAB='AZ' AND rUB.CUI=pUB.CUI AND cAZ.CodeID=AZID AND cUB.SAB='UBERON' RETURN cUB.CodeID+'*'+ tUB.name + '' as UBERONID } -WITH hgnc_id, CLID,UBERONID -RETURN DISTINCT hgnc_id,'cell_types_organ' as ret_key, CLID+ '|' + apoc.text.join(COLLECT(DISTINCT UBERONID),",") AS ret_value -ORDER BY hgnc_id, CLID+ '|' + apoc.text.join(COLLECT(DISTINCT UBERONID),",") +WITH hgnc_id, 'cell_types_organ' as ret_key, CLID,UBERONID, CLID+ '|' + apoc.text.join(COLLECT(DISTINCT UBERONID),",") AS ret_value +RETURN DISTINCT hgnc_id, ret_key, ret_value +ORDER BY hgnc_id, ret_value // Indicate the source of cell type information. UNION @@ -159,4 +162,4 @@ map['cell_types_definition'] AS cell_types_code_definition, map['cell_types_organ'] AS cell_types_codes_organ, map['cell_types_source'] AS cell_types_codes_source -order by hgnc_id \ No newline at end of file +order by hgnc_id diff --git a/src/requirements.txt b/src/requirements.txt index 220c45c..a3eaa15 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,6 +1,7 @@ -ubkg-api==1.4.0 -Flask == 2.1.3 -neo4j == 5.15.0 +# ubkg-api==1.5.0 +git+https://github.com/x-atlas-consortia/ubkg-api.git@neo4jv5 +Flask==2.1.3 +neo4j==5.15.0 # for analysis of tabular data pandas==1.5.0 diff --git a/test_api.sh b/test_api.sh index 0e3976a..e8905ef 100755 --- a/test_api.sh +++ b/test_api.sh @@ -61,6 +61,7 @@ echo "Using UBKG at: ${UBKG_URL}" # Using UBKG at: http://127.0.0.1:5002 echo "assayname_POST..." +echo "SHOULD RETURN 200" curl --request POST \ --url "${UBKG_URL}/assayname" \ --header "Content-Type: application/json" \ @@ -68,48 +69,56 @@ curl --request POST \ echo echo "assaytype GET" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/assaytype?application_context=HUBMAP" \ --header "Accept: application/json" echo echo "assaytype/ GET" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/assaytype/bulk-RNA?application_context=HUBMAP" \ --header "Accept: application/json" echo echo "datasets GET" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/datasets?application_context=HUBMAP" \ --header "Accept: application/json" echo echo "organs GET for HUBMAP" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/organs?application_context=HUBMAP" \ --header "Accept: application/json" echo echo "organs GET for SENNET" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/organs?application_context=SENNET" \ --header "Accept: application/json" echo echo "organs/by-code GET" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/organs/by-code?application_context=HUBMAP" \ --header "Accept: application/json" echo echo "relationships/gene GET..." +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/relationships/gene/MMRN1" \ --header "Content-Type: application/json" echo echo "valueset GET..." +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/valueset?child_sabs=OBI&parent_sab=HUBMAP&parent_code=C001000" \ --header "Content-Type: application/json" @@ -117,16 +126,21 @@ echo # Test for gene_list endpoint echo "genes-info GET" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/genes-info?page=1&genes_per_page=3" \ --header "Content-Type: application/json" echo + echo "genes-info GET last page" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/genes-info?page=last&genes_per_page=3" \ --header "Content-Type: application/json" echo + echo "genes-info GET starts_with B" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/genes-info?genes_per_page=3&starts_with=B" \ --header "Content-Type: application/json" @@ -134,6 +148,7 @@ echo # Test for genes endpoint. echo "genes GET for MMRN1" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/genes/MMRN1" \ --header "Content-Type: application/json" @@ -141,22 +156,29 @@ echo # Test for proteins-info endpoint. echo "proteins-info GET" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/proteins-info?page=1&proteins_per_page=3" \ --header "Content-Type: application/json" echo + echo "proteins-info GET last page" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/proteins-info?page=last&proteins_per_page=3" \ --header "Content-Type: application/json" echo + echo "proteins-info GET starts_with B" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/proteins-info?proteins_per_page=3&starts_with=B" \ --header "Content-Type: application/json" echo + # Test for proteins endpoint. echo "proteins GET for MMRN1_HUMAN" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/proteins/MMRN1_HUMAN" \ --header "Content-Type: application/json" @@ -164,40 +186,50 @@ echo # Test for celltypes list. echo "celltypes-info GET" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/celltypes-info?page=1&proteins_per_page=3" \ --header "Content-Type: application/json" echo + echo "celltypes-info GET last page" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/celltypes-info?page=last&proteins_per_page=3" \ --header "Content-Type: application/json" echo + echo "celltypes-info GET starts_with B" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/celltypes-info?proteins_per_page=3&starts_with=B" \ --header "Content-Type: application/json" echo + # Test for proteins endpoint. echo "celltypes GET for 0002138" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/celltypes/0002138" \ --header "Content-Type: application/json" echo echo "SENNET source types" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/valueset?parent_sab=SENNET&parent_code=C020076&child_sabs=SENNET" \ --header "Content-Type: application/json" echo echo "SENNET sample categories" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/valueset?parent_sab=SENNET&parent_code=C050020&child_sabs=SENNET" \ --header "Content-Type: application/json" echo echo "SENNET entities" +echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/valueset?parent_sab=SENNET&parent_code=C000012&child_sabs=SENNET" \ --header "Content-Type: application/json" @@ -509,4 +541,4 @@ echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-schemas?schema=imc3d" \ --header "Content-Type: application/json" -echo \ No newline at end of file +echo From 982476fc99620ad27e9962ace49f0be4124f8e25 Mon Sep 17 00:00:00 2001 From: ChuckKollar Date: Thu, 1 Feb 2024 13:54:58 -0500 Subject: [PATCH 03/22] Added src/compare_responses.py which allows the user to compare responses from two servers --- src/compare_responses.py | 195 +++++++++++++++++++++++++++++++++++++++ src/requirements.txt | 5 + 2 files changed, 200 insertions(+) create mode 100755 src/compare_responses.py diff --git a/src/compare_responses.py b/src/compare_responses.py new file mode 100755 index 0000000..236aca2 --- /dev/null +++ b/src/compare_responses.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 + +import argparse +import requests +import json +from datatest import validate, ValidationError +# https://zepworks.com/deepdiff/6.7.1/ +from deepdiff import DeepDiff +import pprint + + +UBKG_URL_PROD = 'https://ontology.api.hubmapconsortium.org' +UBKG_URL_DEV = 'https://ontology-api.dev.hubmapconsortium.org' +UBKG_URL_TEST = 'https://ontology-api.test.hubmapconsortium.org' + +URL_LOCAL = 'http://127.0.0.1:5002' + +DIFFERENCES_STATUS_CODE = 0 +DIFFERENCES_RESPONSE = 0 +DIFFERENCES_PROCESSED = 0 +STATUS_CODE_500_HOST_A = 0 +STATUS_CODE_500_HOST_B = 0 + + +class RawTextArgumentDefaultsHelpFormatter( + argparse.ArgumentDefaultsHelpFormatter, + argparse.RawTextHelpFormatter +): + pass + + +# https://docs.python.org/3/howto/argparse.html +parser = argparse.ArgumentParser( + description=""" + Check for differences between endpoints from two UBKG micro-services. + + For example executing: + % src/compare_responses.py https://ontology.api.hubmapconsortium.org http://127.0.0.1:5002 -d + Will compare the results from microservice on ontology.api.hubmapconsortium.org to the + one running on localhost using deepdiff to do the comparison of the output. + + Without using one of '-t', '-d', or '-V' only endpoint status differences will be flagged. + + For documentation on DeepDiff see: https://zepworks.com/deepdiff/6.7.1/diff.html + """, + formatter_class=RawTextArgumentDefaultsHelpFormatter) +parser.add_argument('host_a', type=str, default=UBKG_URL_PROD, + help='host to be compared too (expected)') +parser.add_argument('host_b', type=str, default=URL_LOCAL, + help='host to check for differences') +parser.add_argument("-t", "--textual", action="store_true", + help='show textual differences') +parser.add_argument("-d", "--deepdiff", action="store_true", + help='show differences using DeepDiff') +parser.add_argument("-V", "--validate", action="store_true", + help='show differences using Validate') +parser.add_argument("-v", "--verbose", action="store_true", + help='increase output verbosity') + +args = parser.parse_args() + + +def endpoint_get_diff(host_a: str, host_b: str, path) -> None: + resp_a = requests.get(f"{host_a}{path}") + resp_b = requests.get(f"{host_b}{path}") + resp_diff(host_a, host_b, path, resp_a, resp_b) + + +def endpoint_post_diff(host_a: str, host_b: str, path: str, data: dict) -> None: + resp_a = requests.post(f"{host_a}{path}", json=data) + resp_b = requests.post(f"{host_b}{path}", json=data) + resp_diff(host_a, host_b, path, resp_a, resp_b) + + +def resp_diff(host_a: str, host_b: str, path: str, resp_a, resp_b) -> None: + global DIFFERENCES_STATUS_CODE, DIFFERENCES_RESPONSE, DIFFERENCES_PROCESSED + global STATUS_CODE_500_HOST_A, STATUS_CODE_500_HOST_B + print(f"Checking path: {path} ", end='') + DIFFERENCES_PROCESSED += 1 + if resp_a.status_code == 500: + STATUS_CODE_500_HOST_A += 1 + if resp_b.status_code == 500: + STATUS_CODE_500_HOST_B += 1 + if resp_a.status_code != resp_b.status_code: + print(f"\nSTATUS CODES DIFFER: {host_a}:{resp_a.status_code}; {host_b}:{resp_b.status_code}") + DIFFERENCES_STATUS_CODE += 1 + return + if resp_a.text != resp_b.text: + DIFFERENCES_RESPONSE += 1 + if args.textual is True: + print(f"\nDIFFERENCES (Textual): {host_a}:{resp_a.text}; {host_b}:{resp_b.text}") + resp_a_dict: dict = json.loads(resp_a.text) + resp_b_dict: dict = json.loads(resp_b.text) + if args.deepdiff is True: + # https://zepworks.com/deepdiff/6.7.1/diff.html + deepdiff = DeepDiff(resp_b_dict, resp_a_dict, + ignore_order=True, ignore_string_case=True, + report_repetition=True, verbose_level=2 + ) + print('DIFFERENCE (DeepDiff):') + pp = pprint.PrettyPrinter(indent=4) + pp.pprint(deepdiff) + if args.validate is True: + try: + validate(resp_b_dict, resp_a_dict) + except ValidationError as ve: + # https://datatest.readthedocs.io/en/stable/reference/datatest-core.html#datatest.ValidationError + print('DIFFERENCES (Validate):') + print(f"Expected (host_a): {ve.differences[0].expected}") + print(f"Found (host_b): {ve.differences[0].invalid}") + print() + + return + print('OK!') + + +def diff(host_a: str, host_b: str) -> None: + endpoint_post_diff(host_a, host_b, '/assayname', {"name": "bulk-RNA"}) + endpoint_get_diff(host_a, host_b, '/assaytype?application_context=HUBMAP') + endpoint_get_diff(host_a, host_b, '/assaytype/bulk-RNA?application_context=HUBMAP') + endpoint_get_diff(host_a, host_b, '/datasets?application_context=HUBMAP') + endpoint_get_diff(host_a, host_b, '/organs?application_context=HUBMAP') + endpoint_get_diff(host_a, host_b, '/organs?application_context=SENNET') + endpoint_get_diff(host_a, host_b, '/organs/by-code?application_context=HUBMAP') + endpoint_get_diff(host_a, host_b, '/relationships/gene/MMRN1') + endpoint_get_diff(host_a, host_b, '/valueset?child_sabs=OBI&parent_sab=HUBMAP&parent_code=C001000') + endpoint_get_diff(host_a, host_b, '/genes-info?page=1&genes_per_page=3') + endpoint_get_diff(host_a, host_b, '/genes-info?page=last&genes_per_page=3') + endpoint_get_diff(host_a, host_b, '/genes-info?genes_per_page=3&starts_with=B') + endpoint_get_diff(host_a, host_b, '/genes/MMRN1') + endpoint_get_diff(host_a, host_b, '/proteins-info?page=1&proteins_per_page=3') + endpoint_get_diff(host_a, host_b, '/proteins-info?page=last&proteins_per_page=3') + endpoint_get_diff(host_a, host_b, '/proteins-info?proteins_per_page=3&starts_with=B') + endpoint_get_diff(host_a, host_b, '/proteins/MMRN1_HUMAN') + endpoint_get_diff(host_a, host_b, '/celltypes-info?page=1&proteins_per_page=3') + endpoint_get_diff(host_a, host_b, '/celltypes-info?page=last&proteins_per_page=3') + endpoint_get_diff(host_a, host_b, '/celltypes-info?proteins_per_page=3&starts_with=B') + endpoint_get_diff(host_a, host_b, '/celltypes/0002138') + endpoint_get_diff(host_a, host_b, '/valueset?parent_sab=SENNET&parent_code=C020076&child_sabs=SENNET') + endpoint_get_diff(host_a, host_b, '/valueset?parent_sab=SENNET&parent_code=C050020&child_sabs=SENNET') + endpoint_get_diff(host_a, host_b, '/valueset?parent_sab=SENNET&parent_code=C000012&child_sabs=SENNET') + endpoint_get_diff(host_a, host_b, '/field-descriptions') + endpoint_get_diff(host_a, host_b, '/field-descriptions/acquisition_instrument_model') + endpoint_get_diff(host_a, host_b, '/field-descriptions/acquisition_instrument_model?test=X') + endpoint_get_diff(host_a, host_b, '/field-descriptions/acquisition_instrument_model?source=X') + endpoint_get_diff(host_a, host_b, '/field-descriptions/acquisition_instrument_model?source=HMFIELD') + endpoint_get_diff(host_a, host_b, '/field-types') + endpoint_get_diff(host_a, host_b, '/field-types/acquisition_instrument_model') + endpoint_get_diff(host_a, host_b, '/field-types/acquisition_instrument_model?test=x') + endpoint_get_diff(host_a, host_b, '/field-types/acquisition_instrument_model?mapping_source=X') + endpoint_get_diff(host_a, host_b, '/field-types/acquisition_instrument_model?mapping_source=HMFIELD') + endpoint_get_diff(host_a, host_b, '/field-types/acquisition_instrument_model?type_source=X') + endpoint_get_diff(host_a, host_b, '/field-types/acquisition_instrument_model?type_source=HMFIELD') + endpoint_get_diff(host_a, host_b, '/field-types/acquisition_instrument_model?type=X') + endpoint_get_diff(host_a, host_b, '/field-types/acquisition_instrument_model?type=string') + endpoint_get_diff(host_a, host_b, '/field-types-info') + endpoint_get_diff(host_a, host_b, '/field-types-info?test=x') + endpoint_get_diff(host_a, host_b, '/field-types-info?type_source=x') + endpoint_get_diff(host_a, host_b, '/field-types-info?type_source=HMFIELD') + endpoint_get_diff(host_a, host_b, '/field-assays') + endpoint_get_diff(host_a, host_b, '/field-assays/acquisition_instrument_model') + endpoint_get_diff(host_a, host_b, '/field-assays?test=X') + endpoint_get_diff(host_a, host_b, '/field-assays?assay_identifier=X') + endpoint_get_diff(host_a, host_b, '/field-assays?assay_identifier=snRNAseq') + endpoint_get_diff(host_a, host_b, '/field-assays/acquisition_instrument_model?assay_identifier=snRNAseq') + endpoint_get_diff(host_a, host_b, '/field-assays?data_type=X') + endpoint_get_diff(host_a, host_b, '/field-assays?data_type=seqFISH') + endpoint_get_diff(host_a, host_b, '/field-assays/acquisition_instrument_model?data_type=seqFISH') + endpoint_get_diff(host_a, host_b, '/field-assays?dataset_type=x') + endpoint_get_diff(host_a, host_b, '/field-assays?dataset_type=RNAseq') + endpoint_get_diff(host_a, host_b, '/field-assays/acquisition_instrument_model?dataset_type=RNAseq') + endpoint_get_diff(host_a, host_b, '/field-entities') + endpoint_get_diff(host_a, host_b, '/field-entities?test=X') + endpoint_get_diff(host_a, host_b, '/field-entities/acquisition_instrument_model') + endpoint_get_diff(host_a, host_b, '/field-entities/acquisition_instrument_model?source=x') + endpoint_get_diff(host_a, host_b, '/field-entities/acquisition_instrument_model?source=HMFIELD') + endpoint_get_diff(host_a, host_b, '/field-entities/acquisition_instrument_model?entity=x') + endpoint_get_diff(host_a, host_b, '/field-entities/acquisition_instrument_model?entity=dataset') + endpoint_get_diff(host_a, host_b, '/field-schemas') + endpoint_get_diff(host_a, host_b, '/field-schemas?test=x') + endpoint_get_diff(host_a, host_b, '/field-schemas/acquisition_instrument_model') + endpoint_get_diff(host_a, host_b, '/field-schemas?source=x') + endpoint_get_diff(host_a, host_b, '/field-schemas?source=HMFIELD') + endpoint_get_diff(host_a, host_b, '/field-schemas?schema=x') + endpoint_get_diff(host_a, host_b, '/field-schemas?schema=imc3d') + + +diff(args.host_a, args.host_b) +print(f"\nDifferences processed: {DIFFERENCES_PROCESSED}; " + f"Response Differences: {DIFFERENCES_RESPONSE}" + f"\nStatus Code Differences: {DIFFERENCES_STATUS_CODE}; " + f"Status Codes 500 {args.host_a}: {STATUS_CODE_500_HOST_A}; " + f"Status Codes 500 {args.host_b}: {STATUS_CODE_500_HOST_B} " + ) +print("\nDone!") diff --git a/src/requirements.txt b/src/requirements.txt index a3eaa15..f9ab84c 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -14,3 +14,8 @@ Werkzeug==2.3.7 # Cells API client hubmap-api-py-client==0.0.9 + +# Test and analysis scripts +argparse==1.4.0 +datatest==0.11.1 +deepdiff==6.7.1 From 3b792b90fac448a8ae311388805ee640c54b3953 Mon Sep 17 00:00:00 2001 From: ChuckKollar Date: Fri, 2 Feb 2024 14:39:51 -0500 Subject: [PATCH 04/22] updated DIFFERENCES_PROCESSED > ENDPOINTS_PROCESSED --- src/compare_responses.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/compare_responses.py b/src/compare_responses.py index 236aca2..7fb3b31 100755 --- a/src/compare_responses.py +++ b/src/compare_responses.py @@ -17,7 +17,7 @@ DIFFERENCES_STATUS_CODE = 0 DIFFERENCES_RESPONSE = 0 -DIFFERENCES_PROCESSED = 0 +ENDPOINTS_PROCESSED = 0 STATUS_CODE_500_HOST_A = 0 STATUS_CODE_500_HOST_B = 0 @@ -73,10 +73,10 @@ def endpoint_post_diff(host_a: str, host_b: str, path: str, data: dict) -> None: def resp_diff(host_a: str, host_b: str, path: str, resp_a, resp_b) -> None: - global DIFFERENCES_STATUS_CODE, DIFFERENCES_RESPONSE, DIFFERENCES_PROCESSED + global DIFFERENCES_STATUS_CODE, DIFFERENCES_RESPONSE, ENDPOINTS_PROCESSED global STATUS_CODE_500_HOST_A, STATUS_CODE_500_HOST_B print(f"Checking path: {path} ", end='') - DIFFERENCES_PROCESSED += 1 + ENDPOINTS_PROCESSED += 1 if resp_a.status_code == 500: STATUS_CODE_500_HOST_A += 1 if resp_b.status_code == 500: @@ -186,7 +186,7 @@ def diff(host_a: str, host_b: str) -> None: diff(args.host_a, args.host_b) -print(f"\nDifferences processed: {DIFFERENCES_PROCESSED}; " +print(f"\nEndpoints processed: {ENDPOINTS_PROCESSED}; " f"Response Differences: {DIFFERENCES_RESPONSE}" f"\nStatus Code Differences: {DIFFERENCES_STATUS_CODE}; " f"Status Codes 500 {args.host_a}: {STATUS_CODE_500_HOST_A}; " From 83686c4d699002f7b56bfdf6195b6337c31563b0 Mon Sep 17 00:00:00 2001 From: Derek Furst <85903206+DerekFurstPitt@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:11:53 -0400 Subject: [PATCH 05/22] Create python-app.yml --- .github/workflows/python-app.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/python-app.yml diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml new file mode 100644 index 0000000..c9e91bb --- /dev/null +++ b/.github/workflows/python-app.yml @@ -0,0 +1,26 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Python application +on: + push: + branches: [ "main", "dev-integrate" ] + pull_request: + branches: [ "main", "dev-integrate" ] +permissions: + contents: read +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.9 + uses: actions/setup-python@v3 + with: + python-version: "3.9" + - name: Upgrade Pip + run: python -m pip install --upgrade pip + working-directory: src + - name: Install Dependencies + run: pip install -r requirements.txt + working-directory: src From e336c24cd0e6bc8cb31cb9efd7d4ab4fa93495bb Mon Sep 17 00:00:00 2001 From: AlanSimmons Date: Wed, 13 Mar 2024 17:32:20 -0500 Subject: [PATCH 06/22] enhancement: field_* endpoints account for changes to CEDAR and CEDAR_ENTITY data. --- hs-ontology-api-spec.yaml | 34 +++++-- .../cypher/fieldentities.cypher | 57 ++++++++--- .../fieldentities/fieldentities_controller.py | 17 +++- src/hs_ontology_api/utils/neo4j_logic.py | 99 ++++--------------- test_api.sh | 20 ++-- 5 files changed, 112 insertions(+), 115 deletions(-) diff --git a/hs-ontology-api-spec.yaml b/hs-ontology-api-spec.yaml index f01d91f..7646217 100644 --- a/hs-ontology-api-spec.yaml +++ b/hs-ontology-api-spec.yaml @@ -735,24 +735,33 @@ paths: /field-entities: get: operationId: field_entities_get - summary: Return associations between ingest metadata fields and provenance entities. Replacement for field-entities.yaml. NOTE - available only for fields from field-entities.yaml. + summary: Return associations between ingest metadata fields and provenance entities. Replacement for field-entities.yaml. parameters: - name: source in: query required: false - description: case-insensitive name of the ontology source for the provenance entities (HMFIELD = field-types.yaml; HUBMAP = UBKG) + description: case-insensitive name of the ontology source for the provenance entity mappings. (HMFIELD = from legacy field-entities.yaml) schema: type: string enum: - HMFIELD - - HUBMAP + - CEDAR - name: entity in: query required: false - description: case-sensitive name for entity in either HMFIELD or HUMBMAP ontology + description: case-sensitive name for an entity in either HMFIELD or HUBMAP/SENNET schema: type: string example: Sample + - name: application + in: query + required: false + description: case-insensitive name of the application context + schema: + type: string + enum: + - HUBMAP + - SENNET responses: '200': description: Associations between ingest metadata fields and provenance entities. @@ -761,7 +770,7 @@ paths: schema: $ref: '#/components/schemas/FieldEntitiesResponse' '400': - description: Invalid value for parameter (e.g., *source* not HMFIELD or HUBMAP); invalid parameter + description: Invalid value for parameter (e.g., *source* not HMFIELD or CEDAR); invalid parameter '404': description: No field entities (with list of parameters, if specified) '5XX': @@ -769,7 +778,7 @@ paths: /field-entities/{name}: get: operationId: field_entities_name_get - summary: Return associations between specified ingest metadata field and provenance entities. Replacement for field-entities.yaml. NOTE - available only for fields from field-entities.yaml. + summary: Return associations between specified ingest metadata field and provenance entities. Replacement for field-entities.yaml. parameters: - name: name in: path @@ -781,7 +790,7 @@ paths: - name: source in: query required: false - description: case-insensitive name of the ontology source for the provenance entities (HMFIELD = field-types.yaml; HUBMAP = UBKG) + description: case-insensitive name of the ontology source for the provenance entity mappings. (HMFIELD = from legacy field-entities.yaml) schema: type: string enum: @@ -790,10 +799,19 @@ paths: - name: entity in: query required: false - description: case-sensitive name for entity in either HMFIELD or HUMBMAP ontology + description: case-sensitive name for entity in either HMFIELD or HUBMAP/SENNET schema: type: string example: Sample + - name: application + in: query + required: false + description: case-insensitive name of the application context + schema: + type: string + enum: + - HUBMAP + - SENNET responses: '200': description: Associations between specified ingest metadata field and provenance entities. diff --git a/src/hs_ontology_api/cypher/fieldentities.cypher b/src/hs_ontology_api/cypher/fieldentities.cypher index 0affe75..6bf17cd 100644 --- a/src/hs_ontology_api/cypher/fieldentities.cypher +++ b/src/hs_ontology_api/cypher/fieldentities.cypher @@ -13,11 +13,13 @@ /// Identify all metadata fields, from both: // - legacy sources (the field_*.yaml files in ingest-validation-tools, and modeled in HMFIELD), child codes of HMFIELD:1000 // - current sources (CEDAR tempates, modeled in CEDAR), child codes of CEDAR:TemplateField -// Fields that are in the intersection of HMFIELD and CEDAR share CUIs. // Collect the HMFIELD and CEDAR codes for each metadata field to flatten to level of field name. -// The field_entities_get_logic in neo4j_logic will replace the field_filter and source_filter variables. +// The field_entities_get_logic in neo4j_logic will replace variables that start with the dollar sign. + +// source_filter allows filtering by mapping source (HMFIELD or CEDAR). +// field_filter allows filtering by field name. WITH $source_filter AS source_filter CALL @@ -30,24 +32,49 @@ CALL apoc.text.join(COLLECT(DISTINCT cField.CodeID),'|') AS code_ids, pField.CUI AS CUIField } -// For each field, get associated entities from HMFIELD and HUBMAP ontologies. -// Each HMFIELD entity node is cross-referenced to a HUBMAP entity node. -// (CEDAR fields are currently not associated with entities.) -// The field_entities_get_logic in neo4j_logic will replace the entity_filter and source_filter variables. +// For each field, get associated provenance entities. +// entity_filter allows filtering for provenance entity by name--e.g., "dataset", "Dataset". +// application_filter allows filtering on application context--i.e., "HUBMAP" or "SENNET". + CALL { - WITH CUIField, source_filter - OPTIONAL MATCH (pField:Concept)-[:used_in_entity]->(pEntity:Concept)-[:CODE]->(cHMFIELDEntity:Code)-[rHMFIELD:PT]->(tHMFIELDEntity:Term), - (pEntity:Concept)-[:CODE]->(cHUBMAPEntity:Code)-[rHUBMAP:PT]->(tHUBMAPEntity:Term) - WHERE pField.CUI=CUIField AND cHMFIELDEntity.SAB ='HMFIELD' AND rHMFIELD.CUI=pEntity.CUI - AND cHUBMAPEntity.SAB='HUBMAP' AND rHUBMAP.CUI=pEntity.CUI - $entity_filter - RETURN apoc.text.join([CASE WHEN source_filter in ['HMFIELD',''] THEN REPLACE(cHMFIELDEntity.CodeID,':','|') + '|' + tHMFIELDEntity.name ELSE '' END, - CASE WHEN source_filter in ['HUBMAP',''] THEN REPLACE(cHUBMAPEntity.CodeID,':','|') + '|' + tHUBMAPEntity.name ELSE '' END],';') AS entity + // Each HMFIELD field node is linked to a HMFIELD entity node. + WITH CUIField,source_filter + OPTIONAL MATCH (pField:Concept)-[:used_in_entity]->(pEntity:Concept)-[:CODE]->(cEntity:Code)-[r:PT]->(tEntity:Term) + WHERE pField.CUI=CUIField + AND cEntity.SAB ='HMFIELD' + AND r.CUI=pEntity.CUI + $entity_filter + RETURN DISTINCT CASE WHEN source_filter IN ['HMFIELD',''] THEN REPLACE(cEntity.CodeID,':','|') + '|' + tEntity.name ELSE '' END AS entity + + UNION + + // Each HMFIELD entity node is cross-referenced to HUBMAP and SENNET provenance entity nodes. + WITH CUIField,source_filter + OPTIONAL MATCH (pField:Concept)-[:used_in_entity]->(pEntity:Concept)-[:CODE]->(cEntity:Code)-[r:PT]->(tEntity:Term) + WHERE pField.CUI=CUIField + $application_filter + AND r.CUI=pEntity.CUI + $entity_filter + RETURN DISTINCT CASE WHEN source_filter IN ['HMFIELD',''] THEN REPLACE(cEntity.CodeID,':','|') + '|' + tEntity.name ELSE '' END AS entity + + UNION + + //CEDAR template nodes are mapped to provenance entity nodes in both HUBMAP and SENNET. + //CEDAR field nodes relate to CEDAR template nodes. + WITH CUIField,source_filter + OPTIONAL MATCH (pField:Concept)-[:inverse_has_field]->(pTemplate:Concept)-[:used_in_entity]->(pEntity:Concept)-[:CODE]->(cEntity:Code)-[r:PT]->(tEntity:Term) + WHERE pField.CUI = CUIField + AND r.CUI=pEntity.CUI + //AND c.Entity.SAB in ['HUBMAP','SENNET'] + $application_filter + $entity_filter + RETURN DISTINCT CASE WHEN source_filter IN ['CEDAR',''] THEN REPLACE(cEntity.CodeID,':','|') + '|' + tEntity.name ELSE '' END AS entity } +WITH field_name, code_ids, entity +WHERE entity <>"" WITH field_name, code_ids, COLLECT(entity) AS entities -WHERE entities <>['null;null'] RETURN field_name, code_ids, entities ORDER BY field_name diff --git a/src/hs_ontology_api/routes/fieldentities/fieldentities_controller.py b/src/hs_ontology_api/routes/fieldentities/fieldentities_controller.py index b4d8d7d..b318295 100644 --- a/src/hs_ontology_api/routes/fieldentities/fieldentities_controller.py +++ b/src/hs_ontology_api/routes/fieldentities/fieldentities_controller.py @@ -12,31 +12,40 @@ def field_entities_get(name=None): """Returns detailed information on field entities. + MARCH 2024 - updated for new CEDAR content and CEDAR_ENTITY ontology. + :rtype: Union[List[FieldEntity]] """ # Validate parameters - err = validate_query_parameter_names(['source', 'entity']) + err = validate_query_parameter_names(['source', 'entity','application']) if err != 'ok': return make_response(err, 400) - # Validate mapping source. source = request.args.get('source') if source is not None: source = source.upper() - val_enum = ['HMFIELD', 'HUBMAP'] + val_enum = ['HMFIELD', 'CEDAR'] err = validate_parameter_value_in_enum(param_name='source', param_value=source, enum_list=val_enum) if err != 'ok': return make_response(err, 400) + # Validate application context. + application = request.args.get('application') + if application is not None: + application = application.upper() + val_enum = ['HUBMAP', 'SENNET'] + err = validate_parameter_value_in_enum(param_name='application', param_value=application, enum_list=val_enum) + if err != 'ok': + return make_response(err, 400) entity = request.args.get('entity') neo4j_instance = current_app.neo4jConnectionHelper.instance() result = field_entities_get_logic(neo4j_instance, field_name=name, source=source, - entity=entity) + entity=entity, application=application) if result is None or result == []: # Empty result err = get_404_error_string(prompt_string='No field type associations') diff --git a/src/hs_ontology_api/utils/neo4j_logic.py b/src/hs_ontology_api/utils/neo4j_logic.py index 0631472..a639991 100644 --- a/src/hs_ontology_api/utils/neo4j_logic.py +++ b/src/hs_ontology_api/utils/neo4j_logic.py @@ -1105,21 +1105,7 @@ def field_descriptions_get_logic(neo4j_instance, field_name=None, definition_sou except KeyError: pass - # For queries that are filtered by field, - # 1. Check for multiple rows returned. - # 2. Return the single object instead of an array. - # For non-filtered queries, return an array. - - if field_name is None: - return fielddescriptions - elif record_count > 1: - raise DuplicateFieldError(f"Multiple fields named '{field_name}'.") - elif len(fielddescriptions) == 0: - # No values, so return the empty array. - return fielddescriptions - else: - return fielddescriptions[0] - + return fielddescriptions def field_types_get_logic(neo4j_instance, field_name=None, mapping_source=None, type_source=None, type=None)\ -> List[FieldType]: @@ -1196,21 +1182,7 @@ def field_types_get_logic(neo4j_instance, field_name=None, mapping_source=None, except KeyError: pass - # For queries that are filtered by field, - # 1. Check for multiple rows returned. - # 2. Return the single object instead of an array. - # For non-filtered queries, return an array. - - if field_name is None: - return fieldtypes - elif record_count > 1: - raise DuplicateFieldError(f"Multiple fields named '{field_name}'.") - elif len(fieldtypes) == 0: - # No values, so return the empty array. - return fieldtypes - else: - return fieldtypes[0] - + return fieldtypes def field_types_info_get_logic(neo4j_instance, type_source=None): """ @@ -1337,20 +1309,7 @@ def field_assays_get_logic(neo4j_instance, field_name=None, assay_identifier=Non except KeyError: pass - # For queries that are filtered by field, - # 1. Check for multiple rows returned. - # 2. Return the single object instead of an array. - # For non-filtered queries, return an array. - if field_name is None: - return fieldassays - elif record_count > 1: - raise DuplicateFieldError(f"Multiple fields named '{field_name}'.") - elif len(fieldassays) == 0: - # No values, so return the empty array. - return fieldassays - else: - return fieldassays[0] - + return fieldassays def field_schemas_get_logic(neo4j_instance, field_name=None, mapping_source=None, schema=None) -> List[FieldSchema]: """ @@ -1413,31 +1372,20 @@ def field_schemas_get_logic(neo4j_instance, field_name=None, mapping_source=None except KeyError: pass - # For queries that are filtered by field, - # 1. Check for multiple rows returned. - # 2. Return the single object instead of an array. - # For non-filtered queries, return an array. - if field_name is None: - return fieldschemas - elif record_count > 1: - raise DuplicateFieldError(f"Multiple fields named '{field_name}'.") - elif len(fieldschemas) == 0: - # No values, so return the empty array. - return fieldschemas - else: - return fieldschemas[0] + return fieldschemas -def field_entities_get_logic(neo4j_instance, field_name=None, source=None, entity=None) -> List[FieldEntity]: +def field_entities_get_logic(neo4j_instance, field_name=None, source=None, entity=None, application=None) -> List[FieldEntity]: """ - Returns detailed information on an ingest metadata field's associated data types. - The types here are not to be confused with the dataset data type--e.g., they are values - like "string", "integer", etc. + Returns detailed information on an ingest metadata field's associated entities. + + March 2024 - updated to reflect new CEDAR template data and CEDAR_ENTITY ontology. :param neo4j_instance: neo4j connection to UBKG :param field_name: name of the metadata field - :param source: name of the source of field-entity mapping--i.e., HMFIELD or HUBMAP - :param entity: term for the entity--e.g., string + :param source: name of the source of field-entity mapping--i.e., HMFIELD or CEDAR + :param entity: term for the entity--e.g., "umi_offset" + :param application: application context--i.e., HUBMAP or SENNET """ # response list fieldentities: [FieldEntity] = [] @@ -1460,7 +1408,7 @@ def field_entities_get_logic(neo4j_instance, field_name=None, source=None, entit # Allow for filtering on source. if source is None: source_filter = "''" - elif source in ['HMFIELD', 'HUBMAP']: + elif source in ['HMFIELD', 'CEDAR']: source_filter = f"'{source}'" else: source_filter = "''" @@ -1470,9 +1418,16 @@ def field_entities_get_logic(neo4j_instance, field_name=None, source=None, entit if entity is None: entity_filter = f"AND {identity_filter}" else: - entity_filter = f"AND (tHMFIELDEntity.name='{entity}' OR tHUBMAPEntity.name='{entity}')" + entity_filter = f"AND (tEntity.name='{entity}' OR tEntity.name='{entity}')" query = query.replace('$entity_filter', entity_filter) + # Allow for filtering on application context. + if application is None: + application_filter = f"AND cEntity.SAB IN ['HUBMAP','SENNET']" + else: + application_filter = f"AND cEntity.SAB='{application}'" + query = query.replace('$application_filter', application_filter) + with neo4j_instance.driver.session() as session: # Execute Cypher query. recds: neo4j.Result = session.run(query) @@ -1493,16 +1448,4 @@ def field_entities_get_logic(neo4j_instance, field_name=None, source=None, entit except KeyError: pass - # For queries that are filtered by field, - # 1. Check for multiple rows returned. - # 2. Return the single object instead of an array. - # For non-filtered queries, return an array. - if field_name is None: - return fieldentities - elif record_count > 1: - raise DuplicateFieldError(f"Multiple fields named '{field_name}'.") - elif len(fieldentities) == 0: - # No values, so return the empty array. - return fieldentities - else: - return fieldentities[0] + return fieldentities \ No newline at end of file diff --git a/test_api.sh b/test_api.sh index 0e3976a..c4d40e7 100755 --- a/test_api.sh +++ b/test_api.sh @@ -427,38 +427,38 @@ curl --request GET \ --header "Content-Type: application/json" echo -echo "field-entities/acquisition_instrument_model" +echo "field-entities/umi_offset" echo "SHOULD RETURN 200" curl --request GET \ - --url "${UBKG_URL}/field-entities/acquisition_instrument_model" \ + --url "${UBKG_URL}/field-entities/umi_offset" \ --header "Content-Type: application/json" echo -echo "field-entities/acquisition_instrument_model?source=x" +echo "field-entities/umi_offset?source=x" echo "SHOULD RETURN 400; invalid parameter value" curl --request GET \ - --url "${UBKG_URL}/field-entities/acquisition_instrument_model?source=x" \ + --url "${UBKG_URL}/field-entities/umi_offset?source=x" \ --header "Content-Type: application/json" echo -echo "field-entities/acquisition_instrument_model?source=HMFIELD" +echo "field-entities/umi_offset?source=HMFIELD" echo "SHOULD RETURN 200" curl --request GET \ - --url "${UBKG_URL}/field-entities/acquisition_instrument_model?source=HMFIELD" \ + --url "${UBKG_URL}/field-entities/umi_offset?source=HMFIELD" \ --header "Content-Type: application/json" echo -echo "field-entities/acquisition_instrument_model?entity=x" +echo "field-entities/umi_offset?entity=x" echo "SHOULD RETURN 404; no results" curl --request GET \ - --url "${UBKG_URL}/field-entities/acquisition_instrument_model?entity=x" \ + --url "${UBKG_URL}/field-entities/umi_offset?entity=x" \ --header "Content-Type: application/json" echo -echo "field-entities/acquisition_instrument_model?entity=dataset" +echo "field-entities/umi_offset?entity=dataset" echo "SHOULD RETURN 200" curl --request GET \ - --url "${UBKG_URL}/field-entities/acquisition_instrument_model?entity=dataset" \ + --url "${UBKG_URL}/field-entities/umi_offset?entity=dataset" \ --header "Content-Type: application/json" echo From e5b0c5cc7d881059499733ab4e3c664ddbfcde6d Mon Sep 17 00:00:00 2001 From: AlanSimmons Date: Thu, 14 Mar 2024 11:40:36 -0500 Subject: [PATCH 07/22] renamed application parameter in field-entities to application_context --- hs-ontology-api-spec.yaml | 8 +- src/hs_ontology_api/cypher/fieldassays.cypher | 10 +- .../fieldentities/fieldentities_controller.py | 6 +- test_api.sh | 103 +++++++++--------- 4 files changed, 66 insertions(+), 61 deletions(-) diff --git a/hs-ontology-api-spec.yaml b/hs-ontology-api-spec.yaml index 7646217..5171ea4 100644 --- a/hs-ontology-api-spec.yaml +++ b/hs-ontology-api-spec.yaml @@ -753,7 +753,7 @@ paths: schema: type: string example: Sample - - name: application + - name: application_context in: query required: false description: case-insensitive name of the application context @@ -803,7 +803,7 @@ paths: schema: type: string example: Sample - - name: application + - name: application_context in: query required: false description: case-insensitive name of the application context @@ -828,7 +828,7 @@ paths: /field-assays: get: operationId: field_assays_get - summary: Return associations between ingest metadata fields and the "assays" (dataset data types). Replacement for field-assays.yaml. NOTE only those CEDAR fields that are also in legacy field-assays.yaml can be mapped to assays. + summary: Return associations between ingest metadata fields and the "assays" (dataset data types). Replacement for field-assays.yaml. NOTE only those CEDAR fields that are also in legacy field-assays.yaml can be mapped to assays. In addition, the response from this endpoint is reliable only for datasets that existed prior to the deployment in 2024 of the assay classifier (aka Rules Engine, aka "soft assay types"). parameters: - name: assay_identifier in: query @@ -867,7 +867,7 @@ paths: /field-assays/{name}: get: operationId: field_assays_name_get - summary: Return associations between the specified ingest metadata field and the "assays" (dataset data types). Replacement for field-assays.yaml.NOTE only those CEDAR fields that are also in legacy field-assays.yaml can be mapped to assays. + summary: Return associations between the specified ingest metadata field and the "assays" (dataset data types). Replacement for field-assays.yaml. NOTE only those CEDAR fields that are also in legacy field-assays.yaml can be mapped to assays. parameters: - name: name in: path diff --git a/src/hs_ontology_api/cypher/fieldassays.cypher b/src/hs_ontology_api/cypher/fieldassays.cypher index e225ea6..e1d3ca5 100644 --- a/src/hs_ontology_api/cypher/fieldassays.cypher +++ b/src/hs_ontology_api/cypher/fieldassays.cypher @@ -1,6 +1,9 @@ -// Obtains associations between ingest metadat fields and assay dataset types, both for legacy (HMFIELD) and CEDAR. +// Obtains associations between ingest metadata fields and assay dataset types, both for legacy (HMFIELD) and CEDAR. // Used by the field-assays endpoint. +// NOTE: With the deployment of the assay classifier (Rules Engine, or "soft assay types"), the UBKG is no longer the +// source of truth for assay type. This endpoint is primarily for legacy datasets. + // Identify all metadata fields, from both: // - legacy sources (the field_*.yaml files in ingest-validation-tools, and modeled in HMFIELD), child codes of HMFIELD:1000 // - current sources (CEDAR tempates, modeled in CEDAR), child codes of CEDAR:TemplateField @@ -64,13 +67,14 @@ CALL WITH CUIHMDataset OPTIONAL MATCH (pAssay:Concept)-[:has_data_type]->(pDataType:Concept)-[:CODE]->(cDataType:Code)-[r:PT]->(tDataType:Term) WHERE pAssay.CUI=CUIHMDataset - AND cDataType.SAB='HUBMAP' + AND cDataType.SAB ='HUBMAP' AND r.CUI=pDataType.CUI RETURN CASE WHEN tDataType.name IS NULL THEN 'none' ELSE tDataType.name END AS data_type } // For each HuBMAP Dataset, obtain the "soft assay" dataset type. -// The "soft assay" dataset type is a member of the Soft Assay Dataset Type hierarchy in HUBMAP, with parent code HUBMAP:C003041. +// The "soft assay" dataset type is a member of the Soft Assay Dataset Type hierarchy in HUBMAP, with parent code +// HUBMAP:C003041 CALL { WITH CUIHMDataset diff --git a/src/hs_ontology_api/routes/fieldentities/fieldentities_controller.py b/src/hs_ontology_api/routes/fieldentities/fieldentities_controller.py index b318295..5eccc1d 100644 --- a/src/hs_ontology_api/routes/fieldentities/fieldentities_controller.py +++ b/src/hs_ontology_api/routes/fieldentities/fieldentities_controller.py @@ -19,7 +19,7 @@ def field_entities_get(name=None): """ # Validate parameters - err = validate_query_parameter_names(['source', 'entity','application']) + err = validate_query_parameter_names(['source', 'entity','application_context']) if err != 'ok': return make_response(err, 400) @@ -33,11 +33,11 @@ def field_entities_get(name=None): return make_response(err, 400) # Validate application context. - application = request.args.get('application') + application = request.args.get('application_context') if application is not None: application = application.upper() val_enum = ['HUBMAP', 'SENNET'] - err = validate_parameter_value_in_enum(param_name='application', param_value=application, enum_list=val_enum) + err = validate_parameter_value_in_enum(param_name='application_context', param_value=application, enum_list=val_enum) if err != 'ok': return make_response(err, 400) diff --git a/test_api.sh b/test_api.sh index c4d40e7..77d3fdd 100755 --- a/test_api.sh +++ b/test_api.sh @@ -14,7 +14,7 @@ Help() # Display Help echo "" echo "****************************************" - echo "HELP: UBKG API test script" + echo "HELP: hs-ontology-api API test script" echo echo "Syntax: ./test_api.sh [-option]..." echo "option" @@ -70,151 +70,152 @@ echo echo "assaytype GET" curl --request GET \ --url "${UBKG_URL}/assaytype?application_context=HUBMAP" \ - --header "Accept: application/json" + --header "Accept: application/json" |cut -c1-60 echo + echo "assaytype/ GET" curl --request GET \ --url "${UBKG_URL}/assaytype/bulk-RNA?application_context=HUBMAP" \ - --header "Accept: application/json" + --header "Accept: application/json" |cut -c1-60 echo echo "datasets GET" curl --request GET \ --url "${UBKG_URL}/datasets?application_context=HUBMAP" \ - --header "Accept: application/json" + --header "Accept: application/json" |cut -c1-60 echo echo "organs GET for HUBMAP" curl --request GET \ --url "${UBKG_URL}/organs?application_context=HUBMAP" \ - --header "Accept: application/json" + --header "Accept: application/json" |cut -c1-60 echo echo "organs GET for SENNET" curl --request GET \ --url "${UBKG_URL}/organs?application_context=SENNET" \ - --header "Accept: application/json" + --header "Accept: application/json" |cut -c1-60 echo echo "organs/by-code GET" curl --request GET \ --url "${UBKG_URL}/organs/by-code?application_context=HUBMAP" \ - --header "Accept: application/json" + --header "Accept: application/json" |cut -c1-60 echo echo "relationships/gene GET..." curl --request GET \ --url "${UBKG_URL}/relationships/gene/MMRN1" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "valueset GET..." curl --request GET \ --url "${UBKG_URL}/valueset?child_sabs=OBI&parent_sab=HUBMAP&parent_code=C001000" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo # Test for gene_list endpoint echo "genes-info GET" curl --request GET \ --url "${UBKG_URL}/genes-info?page=1&genes_per_page=3" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "genes-info GET last page" curl --request GET \ --url "${UBKG_URL}/genes-info?page=last&genes_per_page=3" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "genes-info GET starts_with B" curl --request GET \ --url "${UBKG_URL}/genes-info?genes_per_page=3&starts_with=B" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo # Test for genes endpoint. echo "genes GET for MMRN1" curl --request GET \ --url "${UBKG_URL}/genes/MMRN1" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo # Test for proteins-info endpoint. echo "proteins-info GET" curl --request GET \ --url "${UBKG_URL}/proteins-info?page=1&proteins_per_page=3" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "proteins-info GET last page" curl --request GET \ --url "${UBKG_URL}/proteins-info?page=last&proteins_per_page=3" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "proteins-info GET starts_with B" curl --request GET \ --url "${UBKG_URL}/proteins-info?proteins_per_page=3&starts_with=B" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo # Test for proteins endpoint. echo "proteins GET for MMRN1_HUMAN" curl --request GET \ --url "${UBKG_URL}/proteins/MMRN1_HUMAN" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo # Test for celltypes list. echo "celltypes-info GET" curl --request GET \ --url "${UBKG_URL}/celltypes-info?page=1&proteins_per_page=3" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "celltypes-info GET last page" curl --request GET \ --url "${UBKG_URL}/celltypes-info?page=last&proteins_per_page=3" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "celltypes-info GET starts_with B" curl --request GET \ --url "${UBKG_URL}/celltypes-info?proteins_per_page=3&starts_with=B" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo # Test for proteins endpoint. echo "celltypes GET for 0002138" curl --request GET \ --url "${UBKG_URL}/celltypes/0002138" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "SENNET source types" curl --request GET \ --url "${UBKG_URL}/valueset?parent_sab=SENNET&parent_code=C020076&child_sabs=SENNET" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "SENNET sample categories" curl --request GET \ --url "${UBKG_URL}/valueset?parent_sab=SENNET&parent_code=C050020&child_sabs=SENNET" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "SENNET entities" curl --request GET \ --url "${UBKG_URL}/valueset?parent_sab=SENNET&parent_code=C000012&child_sabs=SENNET" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-descriptions" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-descriptions" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-descriptions/acquisition_instrument_model" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-descriptions/acquisition_instrument_model" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-descriptions/acquisition_instrument_model?test=X" @@ -235,21 +236,21 @@ echo "field-descriptions/acquisition_instrument_model?source=HMFIELD" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-descriptions/acquisition_instrument_model?source=HMFIELD" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-types" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-types" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-types/acquisition_instrument_model" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-types/acquisition_instrument_model" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-types/acquisition_instrument_model?test=x" @@ -270,7 +271,7 @@ echo "field-types/acquisition_instrument_model?mapping_source=HMFIELD" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-types/acquisition_instrument_model?mapping_source=HMFIELD" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-types/acquisition_instrument_model?type_source=X" @@ -284,7 +285,7 @@ echo "field-types/acquisition_instrument_model?type_source=HMFIELD" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-types/acquisition_instrument_model?type_source=HMFIELD" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-types/acquisition_instrument_model?type=X" @@ -298,14 +299,14 @@ echo "field-types/acquisition_instrument_model?type=string" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-types/acquisition_instrument_model?type=string" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-types-info" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-types-info" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-types-info?test=x" @@ -326,21 +327,21 @@ echo "field-types-info?type_source=HMFIELD" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-types-info?type_source=HMFIELD" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-assays/" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-assays" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-assays/acquisition_instrument_model" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-assays/acquisition_instrument_model" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-assays/acquisition_instrument_model?test=X" @@ -361,14 +362,14 @@ echo "field-assays?assay_identifier=snRNAseq" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-assays?assay_identifier=snRNAseq" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-assays/acquisition_instrument_model?assay_identifier=snRNAseq" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-assays/acquisition_instrument_model?assay_identifier=snRNAseq" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-assays?data_type=X" @@ -382,14 +383,14 @@ echo "field-assays?data_type=seqFISH" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-assays?data_type=seqFISH" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-assays/acquisition_instrument_model?data_type=seqFISH" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-assays/acquisition_instrument_model?data_type=seqFISH" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-assays?dataset_type=X" @@ -403,21 +404,21 @@ echo "field-assays?dataset_type=RNAseq" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-assays?dataset_type=RNAseq" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-assays/acquisition_instrument_model?dataset_type=RNAseq" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-assays/acquisition_instrument_model?dataset_type=RNAseq" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-entities" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-entities" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-entities?test=X" @@ -431,21 +432,21 @@ echo "field-entities/umi_offset" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-entities/umi_offset" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-entities/umi_offset?source=x" echo "SHOULD RETURN 400; invalid parameter value" curl --request GET \ --url "${UBKG_URL}/field-entities/umi_offset?source=x" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-entities/umi_offset?source=HMFIELD" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-entities/umi_offset?source=HMFIELD" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-entities/umi_offset?entity=x" @@ -459,14 +460,14 @@ echo "field-entities/umi_offset?entity=dataset" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-entities/umi_offset?entity=dataset" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-schemas" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-schemas" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-schemas?test=x" @@ -480,7 +481,7 @@ echo "field-schemas/acquisition_instrument_model" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-schemas/acquisition_instrument_model" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-schemas/acquisition_instrument_model?source=x" @@ -494,7 +495,7 @@ echo "field-schemas/acquisition_instrument_model?source=HMFIELD" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-schemas?source=HMFIELD" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo echo "field-schemas/acquisition_instrument_model?schema=x" @@ -508,5 +509,5 @@ echo "field-schemas/acquisition_instrument_model?schema=imc3d" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-schemas?schema=imc3d" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" |cut -c1-60 echo \ No newline at end of file From 53b7bab18441067808a95bf630b159af82e38c14 Mon Sep 17 00:00:00 2001 From: AlanSimmons Date: Thu, 14 Mar 2024 11:52:43 -0500 Subject: [PATCH 08/22] renamed application parameter in field-entities to application_context --- test_api.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_api.sh b/test_api.sh index 77d3fdd..7ff60e4 100755 --- a/test_api.sh +++ b/test_api.sh @@ -166,17 +166,17 @@ echo # Test for celltypes list. echo "celltypes-info GET" curl --request GET \ - --url "${UBKG_URL}/celltypes-info?page=1&proteins_per_page=3" \ + --url "${UBKG_URL}/celltypes-info?page=1&celltypes_per_page=3" \ --header "Content-Type: application/json" |cut -c1-60 echo echo "celltypes-info GET last page" curl --request GET \ - --url "${UBKG_URL}/celltypes-info?page=last&proteins_per_page=3" \ + --url "${UBKG_URL}/celltypes-info?page=last&celltypes_per_page=3" \ --header "Content-Type: application/json" |cut -c1-60 echo echo "celltypes-info GET starts_with B" curl --request GET \ - --url "${UBKG_URL}/celltypes-info?proteins_per_page=3&starts_with=B" \ + --url "${UBKG_URL}/celltypes-info?celltypes_per_page=3&starts_with=B" \ --header "Content-Type: application/json" |cut -c1-60 echo # Test for proteins endpoint. From 2394ff4f79e3c5648645218438adcfd764030d6a Mon Sep 17 00:00:00 2001 From: AlanSimmons Date: Thu, 14 Mar 2024 11:59:11 -0500 Subject: [PATCH 09/22] updates to test script --- test_api.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_api.sh b/test_api.sh index 7ff60e4..90ad345 100755 --- a/test_api.sh +++ b/test_api.sh @@ -166,17 +166,17 @@ echo # Test for celltypes list. echo "celltypes-info GET" curl --request GET \ - --url "${UBKG_URL}/celltypes-info?page=1&celltypes_per_page=3" \ + --url "${UBKG_URL}/celltypes-info?page=1&cell_types_per_page=3" \ --header "Content-Type: application/json" |cut -c1-60 echo echo "celltypes-info GET last page" curl --request GET \ - --url "${UBKG_URL}/celltypes-info?page=last&celltypes_per_page=3" \ + --url "${UBKG_URL}/celltypes-info?page=last&cell_types_per_page=3" \ --header "Content-Type: application/json" |cut -c1-60 echo echo "celltypes-info GET starts_with B" curl --request GET \ - --url "${UBKG_URL}/celltypes-info?celltypes_per_page=3&starts_with=B" \ + --url "${UBKG_URL}/celltypes-info?cell_types_per_page=3&starts_with=B" \ --header "Content-Type: application/json" |cut -c1-60 echo # Test for proteins endpoint. From 68077f0b89d51767cdfba17d426f910e348302b6 Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Mon, 18 Mar 2024 10:23:29 -0400 Subject: [PATCH 10/22] Bump version to 1.4.11 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ac9f79c..079d7f6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.10 +1.4.11 From d4d938707cc97e545800f89ef6ce3e663264de51 Mon Sep 17 00:00:00 2001 From: AlanSimmons Date: Mon, 18 Mar 2024 10:06:04 -0500 Subject: [PATCH 11/22] updates to test script --- hs-ontology-api-spec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hs-ontology-api-spec.yaml b/hs-ontology-api-spec.yaml index 5171ea4..91cb4f6 100644 --- a/hs-ontology-api-spec.yaml +++ b/hs-ontology-api-spec.yaml @@ -2,7 +2,7 @@ openapi: 3.0.3 info: title: HubMAP/SenNet Ontology API (hs-ontology-api) description: The HuBMAP/SenNet Ontology API contains endpoints for querying a [UBKG](https://ubkg.docs.xconsortia.org/) instance with content from the [HuBMAP/SenNet context](https://ubkg.docs.xconsortia.org/contexts/#hubmapsennet-context). The hs-ontology-api imports the [ubkg-api](https://smart-api.info/ui/96e5b5c0b0efeef5b93ea98ac2794837), which encapsulates both basic connectivity to a UBKG instance and generic endpoint code. - version: 1.4.4 + version: 1.4.11 contact: name: GitHub repository url: https://github.com/x-atlas-consortia/hs-ontology-api From 78016b4a361cd4c7cdf1a2019dafe9b2e360653c Mon Sep 17 00:00:00 2001 From: ChuckKollar Date: Tue, 26 Mar 2024 16:53:49 -0400 Subject: [PATCH 12/22] Removed /status endpoint as it is interited from ubkg-api --- src/main.py | 18 +----------------- src/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/src/main.py b/src/main.py index 91256cc..eb6c59a 100755 --- a/src/main.py +++ b/src/main.py @@ -38,7 +38,7 @@ def make_flask_config(): return temp_flask_app.config -app = UbkgAPI(make_flask_config()).app +app = UbkgAPI(make_flask_config(), Path(__file__).absolute().parent.parent).app app.register_blueprint(assaytype_blueprint) app.register_blueprint(assayname_blueprint) app.register_blueprint(datasets_blueprint) @@ -68,22 +68,6 @@ def make_flask_config(): app.cells_client = OntologyCellsClient(cellsurl) -# Defining the /status endpoint in the ubkg_api package will cause 500 error -# Because the VERSION and BUILD files are not built into the package -@app.route('/status', methods=['GET']) -def api_status(): - status_data = { - # Use strip() to remove leading and trailing spaces, newlines, and tabs - 'version': (Path(__file__).absolute().parent.parent / 'VERSION').read_text().strip(), - 'build': (Path(__file__).absolute().parent.parent / 'BUILD').read_text().strip(), - 'neo4j_connection': False - } - is_connected = current_app.neo4jConnectionHelper.check_connection() - if is_connected: - status_data['neo4j_connection'] = True - - return jsonify(status_data) - #################################################################################################### ## For local development/testing #################################################################################################### diff --git a/src/requirements.txt b/src/requirements.txt index f9ab84c..08e596a 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,4 +1,4 @@ -# ubkg-api==1.5.0 +# ubkg-api==2.0.0 git+https://github.com/x-atlas-consortia/ubkg-api.git@neo4jv5 Flask==2.1.3 neo4j==5.15.0 From 75c4242c6bae7d4c2e7c87b0b171183cd80ce53d Mon Sep 17 00:00:00 2001 From: AlanSimmons Date: Mon, 8 Apr 2024 10:20:18 -0500 Subject: [PATCH 13/22] Updated documentation to reflect working with local or branched version of ubkg-api; moved testing scripts --- .gitignore | 2 ++ README.md | 18 +++++++++++------- {src => test}/compare_responses.py | 0 test_api.sh => test/test_api.sh | 16 +++++++++++++--- 4 files changed, 26 insertions(+), 10 deletions(-) rename {src => test}/compare_responses.py (100%) rename test_api.sh => test/test_api.sh (97%) diff --git a/.gitignore b/.gitignore index cea554d..21bdda4 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,5 @@ docker/ubkg-api/BUILD BUILD **/__pycache__ + +/tests/*/*.out diff --git a/README.md b/README.md index a64d227..ecc3a5a 100644 --- a/README.md +++ b/README.md @@ -55,13 +55,17 @@ If you are modifying code only in hs-ontology-api, you will only need to use the PyPy package version of ubkg-api. The package is included in the requirements.txt file of this repo. If you need to modify both the hs-ontology-api and ubkg-api in concert, you will -need to work with a local instance of the ubkg-api. This is possible by doing the following: -1. Check out a branch of ubkg-api. -2. Configure the local branch of ubkg-api, similarly to the local instance of hs-ontology-api. -3. Start the local instance of ubkg-api. -4. In the virtual environment for hs-ontology-api, install the local instance of ubkg-api using pip with the **-e** flag. This will override the pointer to the ubkg-api package. - -``pip install -e path/to/local/ubkg/repo`` +need to work with a local or branch instance of the ubkg-api. This is possible by doing the following: +1. If your working ubkg-api instance has been committed to a branch, you can point to the branch instance in requirements.txt with a command such as ``git+https://github.com/x-atlas-consortia/ubkg-api.git@`` +2. Check out a branch of ubkg-api. +2. Configure the app.cfg file of the local branch of ubkg-api to connect to the appropriate UBKG instance. +3. In the virtual environment for hs-ontology-api, install an editable local instance of ubkg-api. Two ways to do this: + a. ``pip install -e path/to/local/ubkg-api/repo`` + b. If using PyCharm, in the **Python Packages** tab, + 1) Click **Add Package**. + 2) Navigate to the root of the ubkg-api repo. + 3) Indicate that the package is editable. +4. Because ubkg-api has a PyPI TOML file, any of the aforementioned commands will compile a local package and override the pointer to the ubkg-api package. ## Connecting to the local instance of hs-ontology-api For URLs that execute endpoints in your local instance, use the values indicated in the **main.py** script, in the section prefaced with the comment `For local development/testing`: diff --git a/src/compare_responses.py b/test/compare_responses.py similarity index 100% rename from src/compare_responses.py rename to test/compare_responses.py diff --git a/test_api.sh b/test/test_api.sh similarity index 97% rename from test_api.sh rename to test/test_api.sh index e8905ef..583b53f 100755 --- a/test_api.sh +++ b/test/test_api.sh @@ -55,18 +55,28 @@ case "$env" in esac echo "Using UBKG at: ${UBKG_URL}" +echo "Only the first 60 characters of output from HTTP 200 returns displayed." + # $ ./test_api.sh # Using UBKG at: https://ontology-api.dev.hubmapconsortium.org # $ (export UBKG_URL=http://127.0.0.1:5002; ./test_api.sh) # Using UBKG at: http://127.0.0.1:5002 -echo "assayname_POST..." -echo "SHOULD RETURN 200" +#-------------------------------------------- +echo "TESTS FOR: assayname POST" | tee -a test.out +echo "SIGNATURE: /assayname" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out + +echo "/assayname_POST: should return 200" | tee -a test.out curl --request POST \ --url "${UBKG_URL}/assayname" \ --header "Content-Type: application/json" \ - --data '{"name": "bulk-RNA"}' + --data '{"name": "bulk-RNA"}' | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out +exit; echo "assaytype GET" echo "SHOULD RETURN 200" From ffbbf2feb2a487c74e91ed9d049ebde207a630f2 Mon Sep 17 00:00:00 2001 From: AlanSimmons Date: Tue, 9 Apr 2024 07:01:22 -0500 Subject: [PATCH 14/22] gitignore to ignore any files generated by prototype cells_index script --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 21bdda4..780906a 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,5 @@ BUILD **/__pycache__ /tests/*/*.out +/src/cells_index/*.csv +/src/cells_index/*.tsv \ No newline at end of file From b739eb2d6d9ed8ac026e4c71a4c23431c437cf79 Mon Sep 17 00:00:00 2001 From: AlanSimmons Date: Tue, 9 Apr 2024 11:06:37 -0500 Subject: [PATCH 15/22] Refactored test script to write to output file. --- test/test_api.sh | 575 ++++++++++++++++++++++++++++++----------------- 1 file changed, 363 insertions(+), 212 deletions(-) diff --git a/test/test_api.sh b/test/test_api.sh index 583b53f..42f18c8 100755 --- a/test/test_api.sh +++ b/test/test_api.sh @@ -54,8 +54,11 @@ case "$env" in esac -echo "Using UBKG at: ${UBKG_URL}" -echo "Only the first 60 characters of output from HTTP 200 returns displayed." +echo "Using UBKG at: ${UBKG_URL}" | tee test.out +echo "For these tests, only first 60 characters of output from HTTP 200 returns displayed." | tee -a test.out +echo "To review response bodies in detail, call endpoints individually." | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out # $ ./test_api.sh # Using UBKG at: https://ontology-api.dev.hubmapconsortium.org @@ -68,7 +71,7 @@ echo "SIGNATURE: /assayname" | tee -a test.out echo | tee -a test.out echo | tee -a test.out -echo "/assayname_POST: should return 200" | tee -a test.out +echo "/assayname_POST with bulk-RNA => should return 200" | tee -a test.out curl --request POST \ --url "${UBKG_URL}/assayname" \ --header "Content-Type: application/json" \ @@ -76,479 +79,627 @@ curl --request POST \ echo echo | tee -a test.out echo | tee -a test.out -exit; -echo "assaytype GET" -echo "SHOULD RETURN 200" +echo "TESTS FOR: assaytypes GET" | tee -a test.out +echo "SIGNATURE: /assaytypes?application_context=" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out +echo "/assaytype?application_context=HUBMAP GET => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/assaytype?application_context=HUBMAP" \ - --header "Accept: application/json" + --header "Accept: application/json"| cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "assaytype/ GET" -echo "SHOULD RETURN 200" + +echo "TESTS FOR: assaytypes GET" | tee -a test.out +echo "SIGNATURE: /assaytypes/?application_context=" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out +echo "/assaytypes/bulk-RNA?application_context=HUBMAP => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/assaytype/bulk-RNA?application_context=HUBMAP" \ - --header "Accept: application/json" + --header "Accept: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out + +echo "TESTS FOR: datasets GET" | tee -a test.out +echo "SIGNATURE: /datasets?application_context=&data_type=&description=&alt_name=&primary=&contains_pii=&vis_only=&vitessce_hint=&dataset_provider=" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out +echo "/datasets?application_context=HUBMAP => should return 200" | tee -a test.out -echo "datasets GET" -echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/datasets?application_context=HUBMAP" \ - --header "Accept: application/json" + --header "Accept: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "organs GET for HUBMAP" -echo "SHOULD RETURN 200" + +echo "TESTS FOR: organs GET" | tee -a test.out +echo "SIGNATURE: /organs?application_context=" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out + +echo "1. /organs => missing application context; should return custom 400" | tee -a test.out +curl --request GET \ + --url "${UBKG_URL}/organs" \ + --header "Accept: application/json" | cut -c1-60 | tee -a test.out +echo +echo | tee -a test.out +echo | tee -a test.out + +echo "2. /organs?application_context=HUBMAP => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/organs?application_context=HUBMAP" \ - --header "Accept: application/json" + --header "Accept: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "organs GET for SENNET" -echo "SHOULD RETURN 200" +echo "3. /organs?application_context=SENNET => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/organs?application_context=SENNET" \ - --header "Accept: application/json" + --header "Accept: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "organs/by-code GET" -echo "SHOULD RETURN 200" +echo "TESTS FOR: organs/by-code GET" | tee -a test.out +echo "SIGNATURE: /organs/by-code?application_context=" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out + +echo "/organs/by-code?application_context=HUBMAP => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/organs/by-code?application_context=HUBMAP" \ - --header "Accept: application/json" + --header "Accept: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "relationships/gene GET..." -echo "SHOULD RETURN 200" +echo "TESTS FOR: relationships/gene GET" | tee -a test.out +echo "SIGNATURE: /relationships/gene/" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out + +echo "relationships/gene/MMRN1 => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/relationships/gene/MMRN1" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json"| cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "valueset GET..." -echo "SHOULD RETURN 200" +echo "TESTS FOR: valueset GET" | tee -a test.out +echo "SIGNATURE: /valueset?child_sabs=&parent_code=" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out + +echo "/valueset?child_sabs=OBI&parent_sab=HUBMAP&parent_code=C001000 => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/valueset?child_sabs=OBI&parent_sab=HUBMAP&parent_code=C001000" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -# Test for gene_list endpoint -echo "genes-info GET" -echo "SHOULD RETURN 200" +echo "TESTS FOR: genes-info GET" | tee -a test.out +echo "SIGNATURE: /gene_list?page=&genes_per_page=&starts_with=" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out + +# Test for genes-info endpoint +echo "1. /genes-info?page=1&genes_per_page=3 => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/genes-info?page=1&genes_per_page=3" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "genes-info GET last page" -echo "SHOULD RETURN 200" +echo "2. /genes-info?page=last&genes_per_page=3 => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/genes-info?page=last&genes_per_page=3" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json"| cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "genes-info GET starts_with B" -echo "SHOULD RETURN 200" +echo "3. /genes-info?genes_per_page=3&starts_with=B => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/genes-info?genes_per_page=3&starts_with=B" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out # Test for genes endpoint. -echo "genes GET for MMRN1" -echo "SHOULD RETURN 200" +echo "TESTS FOR: genes GET" | tee -a test.out +echo "SIGNATURE: /genes/" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out + +echo "/genes/MMRN1 => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/genes/MMRN1" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out # Test for proteins-info endpoint. -echo "proteins-info GET" -echo "SHOULD RETURN 200" +echo "TESTS FOR: proteins-info GET" | tee -a test.out +echo "SIGNATURE: /proteins_info?page=&proteins_per_page=&starts_with=" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out + +echo "1. /proteins-info?page=1&proteins_per_page=3 => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/proteins-info?page=1&proteins_per_page=3" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json"| cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "proteins-info GET last page" +echo "2. /proteins-info?page=last&proteins_per_page=3 => should return 200" | tee -a test.out echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/proteins-info?page=last&proteins_per_page=3" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "proteins-info GET starts_with B" -echo "SHOULD RETURN 200" +echo "3. /proteins-info?proteins_per_page=3&starts_with=B => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/proteins-info?proteins_per_page=3&starts_with=B" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out + +echo "TESTS FOR: proteins GET" | tee -a test.out +echo "SIGNATURE: /genes/" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out # Test for proteins endpoint. -echo "proteins GET for MMRN1_HUMAN" +echo "/proteins/MMRN1_HUMAN => should return 200" | tee -a test.out echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/proteins/MMRN1_HUMAN" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -# Test for celltypes list. -echo "celltypes-info GET" -echo "SHOULD RETURN 200" +echo "TESTS FOR: celltypes-info GET" | tee -a test.out +echo "SIGNATURE: /celltypesinfo?page=&cell_types_per_page=&starts_with=" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out + +echo "1. /celltypes-info?page=1&proteins_per_page=3 => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/celltypes-info?page=1&proteins_per_page=3" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "celltypes-info GET last page" -echo "SHOULD RETURN 200" +echo "2. /celltypes-info?page=last&proteins_per_page=3 => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/celltypes-info?page=last&proteins_per_page=3" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "celltypes-info GET starts_with B" -echo "SHOULD RETURN 200" +echo "2. /celltypes-info?proteins_per_page=3&starts_with=B => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/celltypes-info?proteins_per_page=3&starts_with=B" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -# Test for proteins endpoint. -echo "celltypes GET for 0002138" -echo "SHOULD RETURN 200" +# Test for celltypes endpoint. +echo "TESTS FOR: celltypes GET" | tee -a test.out +echo "SIGNATURE: /celltypesinfo/" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out + +echo "/celltypes/0002138 => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/celltypes/0002138" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "SENNET source types" -echo "SHOULD RETURN 200" +echo "TEST FOR: SENNET source types" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/valueset?parent_sab=SENNET&parent_code=C020076&child_sabs=SENNET" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "SENNET sample categories" -echo "SHOULD RETURN 200" +echo "TEST FOR: SENNET sample categories" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/valueset?parent_sab=SENNET&parent_code=C050020&child_sabs=SENNET" \ - --header "Content-Type: application/json" -echo + --header "Content-Type: application/json" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out -echo "SENNET entities" -echo "SHOULD RETURN 200" +echo "TEST FOR: SENNET entities" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/valueset?parent_sab=SENNET&parent_code=C000012&child_sabs=SENNET" \ - --header "Content-Type: application/json" -echo + --header "Content-Type: application/json" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out -echo "field-descriptions" -echo "SHOULD RETURN 200" +echo "TESTS FOR: field-descriptions GET" | tee -a test.out +echo "SIGNATURE: /field-descriptions/?source=" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out + +echo "1./ field-descriptions => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-descriptions" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-descriptions/acquisition_instrument_model" -echo "SHOULD RETURN 200" +echo "2. /field-descriptions/acquisition_instrument_model => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-descriptions/acquisition_instrument_model" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-descriptions/acquisition_instrument_model?test=X" -echo "SHOULD RETURN 400; invalid parameter" +echo "3. /field-descriptions/acquisition_instrument_model?test=X => invalid parameter; should return custom 400" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-descriptions/acquisition_instrument_model?test=X" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-descriptions/acquisition_instrument_model?source=X" +echo "4. /field-descriptions/acquisition_instrument_model?source=X => invalid source; should return custom 400" | tee -a test.out echo "SHOULD RETURN 400; invalid parameter value" curl --request GET \ --url "${UBKG_URL}/field-descriptions/acquisition_instrument_model?source=X" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-descriptions/acquisition_instrument_model?source=HMFIELD" -echo "SHOULD RETURN 200" +echo "5. field-descriptions/acquisition_instrument_model?source=HMFIELD = > should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-descriptions/acquisition_instrument_model?source=HMFIELD" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-types" -echo "SHOULD RETURN 200" +echo "TESTS FOR: field-types GET" | tee -a test.out +echo "SIGNATURE: /field-types/?mapping_source=&type_source=&type=" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out + +echo "1. /field-types => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-types" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-types/acquisition_instrument_model" +echo "2. /field-types/acquisition_instrument_model => should return 200" | tee -a test.out echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-types/acquisition_instrument_model" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-types/acquisition_instrument_model?test=x" -echo "SHOULD RETURN 400; invalid parameter" +echo "3. field-types/acquisition_instrument_model?test=x => invalid parameter: should return 400" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-types/acquisition_instrument_model?test=x" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-types/acquisition_instrument_model?mapping_source=X" -echo "SHOULD RETURN 400; invalid parameter value" +echo "4. field-types/acquisition_instrument_model?mapping_source=X => invalid mapping source; should return 400" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-types/acquisition_instrument_model?mapping_source=X" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-types/acquisition_instrument_model?mapping_source=HMFIELD" -echo "SHOULD RETURN 200" +echo "5. field-types/acquisition_instrument_model?mapping_source=HMFIELD => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-types/acquisition_instrument_model?mapping_source=HMFIELD" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-types/acquisition_instrument_model?type_source=X" -echo "SHOULD RETURN 400; invalid parameter value" +echo "6. field-types/acquisition_instrument_model?type_source=X => invalid type_source; should return 400" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-types/acquisition_instrument_model?type_source=X" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-types/acquisition_instrument_model?type_source=HMFIELD" -echo "SHOULD RETURN 200" +echo "7. field-types/acquisition_instrument_model?type_source=HMFIELD => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-types/acquisition_instrument_model?type_source=HMFIELD" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-types/acquisition_instrument_model?type=X" -echo "SHOULD RETURN 404; no results" +echo "8. field-types/acquisition_instrument_model?type=X => invalid type; should return 404" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-types/acquisition_instrument_model?type=X" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-types/acquisition_instrument_model?type=string" -echo "SHOULD RETURN 200" +echo "9. field-types/acquisition_instrument_model?type=string => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-types/acquisition_instrument_model?type=string" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-types-info" -echo "SHOULD RETURN 200" +echo "TESTS FOR: field-types-info GET" | tee -a test.out +echo "SIGNATURE: /field-types-info?type_source=" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out + +echo "1. /field-types-info => SHOULD RETURN 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-types-info" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-types-info?test=x" -echo "SHOULD RETURN 400; invalid parameter" +echo "2. /field-types-info?test=x => invalid parameter; should return 400" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-types-info?test=x" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-types-info?type_source=x" -echo "SHOULD RETURN 400; invalid parameter value" +echo "3. /field-types-info?type_source=x => invalid type_source; should return 400" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-types-info?type_source=x" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-types-info?type_source=HMFIELD" -echo "SHOULD RETURN 200" +echo "4. field-types-info?type_source=HMFIELD => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-types-info?type_source=HMFIELD" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-assays/" -echo "SHOULD RETURN 200" +echo "TESTS FOR: field-assays GET" | tee -a test.out +echo "SIGNATURE: /field-assays/?data_type=&dataset_type= should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-assays" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-assays/acquisition_instrument_model" -echo "SHOULD RETURN 200" +echo "2. /field-assays/acquisition_instrument_model => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-assays/acquisition_instrument_model" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-assays/acquisition_instrument_model?test=X" -echo "SHOULD RETURN 400; invalid parameter" +echo "3. field-assays/acquisition_instrument_model?test=X => invalid parameter; should return 400" | tee -a test.out curl --request GET \ - --url "${UBKG_URL}/field-assays?test=X" \ - --header "Content-Type: application/json" + --url "${UBKG_URL}/field-assays/acquisition_instrument_model?test=X" \ + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-assays?assay_identifier=X" +echo "4. /field-assays?assay_identifier=X => no results; should return 404" | tee -a test.out echo "SHOULD RETURN 404; no results" curl --request GET \ --url "${UBKG_URL}/field-assays?assay_identifier=X" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-assays?assay_identifier=snRNAseq" -echo "SHOULD RETURN 200" +echo "5. /field-assays?assay_identifier=snRNAseq => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-assays?assay_identifier=snRNAseq" \ - --header "Content-Type: application/json" -echo - -echo "field-assays/acquisition_instrument_model?assay_identifier=snRNAseq" -echo "SHOULD RETURN 200" -curl --request GET \ - --url "${UBKG_URL}/field-assays/acquisition_instrument_model?assay_identifier=snRNAseq" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-assays?data_type=X" -echo "SHOULD RETURN 404; no results" +echo "5. /field-assays?data_type=X => no results; should return 404" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-assays?data_type=X" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-assays?data_type=seqFISH" -echo "SHOULD RETURN 200" +echo "6. /field-assays?data_type=seqFISH => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-assays?data_type=seqFISH" \ - --header "Content-Type: application/json" -echo - -echo "field-assays/acquisition_instrument_model?data_type=seqFISH" -echo "SHOULD RETURN 200" -curl --request GET \ - --url "${UBKG_URL}/field-assays/acquisition_instrument_model?data_type=seqFISH" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-assays?dataset_type=X" -echo "SHOULD RETURN 404; no results" +echo "7. field-assays?dataset_type=X => no results; should return 404" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-assays?dataset_type=x" \ --header "Content-Type: application/json" echo -echo "field-assays?dataset_type=RNAseq" -echo "SHOULD RETURN 200" +echo "8. /field-assays?dataset_type=RNAseq => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-assays?dataset_type=RNAseq" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json"| cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-assays/acquisition_instrument_model?dataset_type=RNAseq" -echo "SHOULD RETURN 200" -curl --request GET \ - --url "${UBKG_URL}/field-assays/acquisition_instrument_model?dataset_type=RNAseq" \ - --header "Content-Type: application/json" -echo +echo "TESTS FOR: field-entities GET" | tee -a test.out +echo "SIGNATURE: /field-entities/?source=&entity=&application_context=" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out -echo "field-entities" -echo "SHOULD RETURN 200" +echo "1. /field-entities => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-entities" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-entities?test=X" -echo "SHOULD RETURN 400; invalid parameter" +echo "2. /field-entities?test=X => invalid parameter; should return 400" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-entities?test=X" \ --header "Content-Type: application/json" echo -echo "field-entities/acquisition_instrument_model" -echo "SHOULD RETURN 200" +echo "3. /field-entities/acquisition_instrument_model => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-entities/acquisition_instrument_model" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-entities/acquisition_instrument_model?source=x" -echo "SHOULD RETURN 400; invalid parameter value" +echo "4. /field-entities/acquisition_instrument_model?source=x => invalid source; should return 400" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-entities/acquisition_instrument_model?source=x" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-entities/acquisition_instrument_model?source=HMFIELD" -echo "SHOULD RETURN 200" +echo "5. /field-entities/acquisition_instrument_model?source=HMFIELD => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-entities/acquisition_instrument_model?source=HMFIELD" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-entities/acquisition_instrument_model?entity=x" -echo "SHOULD RETURN 404; no results" +echo "6. /field-entities/acquisition_instrument_model?entity=x => no results; should return 404" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-entities/acquisition_instrument_model?entity=x" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-entities/acquisition_instrument_model?entity=dataset" +echo "7. /field-entities/acquisition_instrument_model?entity=dataset => should return 200" echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-entities/acquisition_instrument_model?entity=dataset" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-schemas" -echo "SHOULD RETURN 200" +echo "TESTS FOR: field-schemas GET" | tee -a test.out +echo "SIGNATURE: /field-entities/?source=&schema=" | tee -a test.out +echo | tee -a test.out +echo | tee -a test.out + +echo "1. /field-schemas => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-schemas" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-schemas?test=x" -echo "SHOULD RETURN 400; invalid parameter" +echo "2. /field-schemas?test=x => invalid parameter; should return 400" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-schemas?test=x" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-schemas/acquisition_instrument_model" -echo "SHOULD RETURN 200" + +echo "3. /field-schemas/acquisition_instrument_model => should return 200" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-schemas/acquisition_instrument_model" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-schemas/acquisition_instrument_model?source=x" -echo "SHOULD RETURN 400; invalid parameter value" +echo "4. /field-schemas/acquisition_instrument_model?source=x => invalid parameter value; should return 400" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-schemas?source=x" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-schemas/acquisition_instrument_model?source=HMFIELD" +echo "5. /field-schemas/acquisition_instrument_model?source=HMFIELD => should return 200" | tee -a test.out echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-schemas?source=HMFIELD" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-schemas/acquisition_instrument_model?schema=x" -echo "SHOULD RETURN 404; no results" +echo "6. /field-schemas/acquisition_instrument_model?schema=x => no results; should return 404" | tee -a test.out curl --request GET \ --url "${UBKG_URL}/field-schemas?schema=x" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out echo +echo | tee -a test.out +echo | tee -a test.out -echo "field-schemas/acquisition_instrument_model?schema=imc3d" +echo "7. /field-schemas/acquisition_instrument_model?schema=imc3d => should return 200" | tee -a test.out echo "SHOULD RETURN 200" curl --request GET \ --url "${UBKG_URL}/field-schemas?schema=imc3d" \ - --header "Content-Type: application/json" + --header "Content-Type: application/json" | cut -c1-60 | tee -a test.out +echo +echo | tee -a test.out +echo | tee -a test.out echo From 3551b30f2993ad278fe27aa4381d3794515bec05 Mon Sep 17 00:00:00 2001 From: AlanSimmons Date: Tue, 9 Apr 2024 15:07:46 -0500 Subject: [PATCH 16/22] Bug fixes in genedetail.cypher: 1. HRA changed label for "has_marker_component" to "characterized_by" 2. Exception for case of invalid input parameter--i.e., invalid HGNC symbol or code --- src/hs_ontology_api/cypher/genedetail.cypher | 21 ++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/hs_ontology_api/cypher/genedetail.cypher b/src/hs_ontology_api/cypher/genedetail.cypher index 315a6b6..d45fc4b 100644 --- a/src/hs_ontology_api/cypher/genedetail.cypher +++ b/src/hs_ontology_api/cypher/genedetail.cypher @@ -74,8 +74,9 @@ ORDER BY hgnc_id,ret_key UNION //Cell types - CL Codes +// APRIL 2024 - HRA changed "has_marker_component" to "characterized_by" WITH GeneCUI -OPTIONAL MATCH (cGene:Code)<-[:CODE]-(pGene:Concept)-[:inverse_has_marker_component]->(pCL:Concept)-[:CODE]->(cCL:Code)-[rCL]->(tCL:Term) WHERE pGene.CUI=GeneCUI AND cGene.SAB='HGNC' AND cCL.SAB='CL' AND rCL.CUI=pCL.CUI RETURN toInteger(cGene.CODE) AS hgnc_id, 'cell_types_code' AS ret_key, cCL.CodeID AS ret_value +OPTIONAL MATCH (cGene:Code)<-[:CODE]-(pGene:Concept)-[:inverse_characterized_by]->(pCL:Concept)-[:CODE]->(cCL:Code)-[rCL]->(tCL:Term) WHERE pGene.CUI=GeneCUI AND cGene.SAB='HGNC' AND cCL.SAB='CL' AND rCL.CUI=pCL.CUI RETURN toInteger(cGene.CODE) AS hgnc_id, 'cell_types_code' AS ret_key, cCL.CodeID AS ret_value ORDER BY hgnc_id,ret_key,ret_value UNION @@ -87,13 +88,15 @@ UNION // The preferred term will be the term of type PT; if there is no PT, then any of the others of type PT_SAB will do. // First, order the preferred terms by whether they are the PT or a PT_SAB. +// APRIL 2024 - HRA changed the label from "has_marker_component" to "characterized_by" WITH GeneCUI CALL{ WITH GeneCUI -OPTIONAL MATCH (cGene:Code)<-[:CODE]-(pGene:Concept)-[:inverse_has_marker_component]->(pCL:Concept)-[:CODE]->(cCL:Code)-[rCL]->(tCL:Term) WHERE pGene.CUI=GeneCUI AND cGene.SAB='HGNC' AND cCL.SAB='CL' AND rCL.CUI=pCL.CUI AND type(rCL) STARTS WITH 'PT' RETURN toInteger(cGene.CODE) AS hgnc_id, cCL.CodeID AS CLID, MIN(CASE WHEN type(rCL)='PT' THEN 0 ELSE 1 END) AS mintype order by hgnc_id,CLID,mintype +OPTIONAL MATCH (cGene:Code)<-[:CODE]-(pGene:Concept)-[:inverse_characterized_by]->(pCL:Concept)-[:CODE]->(cCL:Code)-[rCL]->(tCL:Term) WHERE pGene.CUI=GeneCUI AND cGene.SAB='HGNC' AND cCL.SAB='CL' AND rCL.CUI=pCL.CUI AND type(rCL) STARTS WITH 'PT' RETURN toInteger(cGene.CODE) AS hgnc_id, cCL.CodeID AS CLID, MIN(CASE WHEN type(rCL)='PT' THEN 0 ELSE 1 END) AS mintype order by hgnc_id,CLID,mintype } // Next, filter to either the PT or one of the PT_SABs. +// MARCH 2024 - WITH used in return to upgrade to v5 Cypher. WITH hgnc_id, CLID, mintype OPTIONAL MATCH (cCL:Code)-[rCL]->(tCL:Term) where cCL.CodeID = CLID AND type(rCL) STARTS WITH 'PT' @@ -106,8 +109,10 @@ UNION // Cell types - CL code|definition // Definitions link to Concepts and multiple CL codes can match to the same concept; however, each CL code has a "preferred" CUI, identified by the CUI property of the relationship of any of the code's linked terms. +// MARCH 2024 - final WITH added to work with v5 Cypher +// APRIL 2024 - HRA changed "has_marker_component" to "characterized_by" WITH GeneCUI -OPTIONAL MATCH (cGene:Code)<-[:CODE]-(pGene:Concept)-[:inverse_has_marker_component]->(pCL:Concept)-[:CODE]->(cCL:Code)-[rCL]->(tCL:Term),(pCL:Concept)-[:DEF]->(dCL:Definition) WHERE rCL.CUI=pCL.CUI AND pGene.CUI=GeneCUI AND cGene.SAB='HGNC' AND cCL.SAB='CL' AND dCL.SAB='CL' +OPTIONAL MATCH (cGene:Code)<-[:CODE]-(pGene:Concept)-[:inverse_characterized_by]->(pCL:Concept)-[:CODE]->(cCL:Code)-[rCL]->(tCL:Term),(pCL:Concept)-[:DEF]->(dCL:Definition) WHERE rCL.CUI=pCL.CUI AND pGene.CUI=GeneCUI AND cGene.SAB='HGNC' AND cCL.SAB='CL' AND dCL.SAB='CL' WITH toInteger(cGene.CODE) AS hgnc_id,'cell_types_definition' as ret_key, cCL.CodeID + '|'+ dCL.DEF as ret_value RETURN DISTINCT hgnc_id, ret_key, ret_value ORDER BY hgnc_id, ret_value @@ -121,11 +126,12 @@ UNION // 3. Assigns UBERON codes as cross-references to AZ organ codes. // // To get organ information, map gene to cell type to organ location. +// APRIL 2024 - HRA changed "has_marker_component" to "characterized_by" WITH GeneCUI //First, get Azimuth Codes that are cross-referenced to CL codes. For the case of a CL code being cross-referenced to multiple AZ codes, only one AZ code gets the "preferred" cross-reference to the CL code; however, all AZ codes have a cross-reference to the CL code, so do not check on rAZ.CUI=pCL.CUI. CALL {WITH GeneCUI -OPTIONAL MATCH (cGene:Code)<-[:CODE]-(pGene:Concept)-[:inverse_has_marker_component]->(pCL:Concept)-[:CODE]->(cCL:Code)-[rCL]->(tCL:Term), (pCL:Concept)-[:CODE]->(cAZ:Code)-[rAZ]->(tAZ:Term) WHERE rCL.CUI=pCL.CUI AND pGene.CUI=GeneCUI AND cGene.SAB='HGNC' AND cCL.SAB='CL' AND cAZ.SAB='AZ' RETURN DISTINCT toInteger(cGene.CODE) AS hgnc_id,cCL.CodeID as CLID,cAZ.CodeID AS AZID} +OPTIONAL MATCH (cGene:Code)<-[:CODE]-(pGene:Concept)-[:inverse_characterized_by]->(pCL:Concept)-[:CODE]->(cCL:Code)-[rCL]->(tCL:Term), (pCL:Concept)-[:CODE]->(cAZ:Code)-[rAZ]->(tAZ:Term) WHERE rCL.CUI=pCL.CUI AND pGene.CUI=GeneCUI AND cGene.SAB='HGNC' AND cCL.SAB='CL' AND cAZ.SAB='AZ' RETURN DISTINCT toInteger(cGene.CODE) AS hgnc_id,cCL.CodeID as CLID,cAZ.CodeID AS AZID} //Use the AZ codes to map to concepts that have located_in relationships with AZ organ codes. The AZ organ codes are cross-referenced to UBERON codes. Limit the located_in relationships to those from AZ. CALL {WITH AZID @@ -137,16 +143,19 @@ RETURN DISTINCT hgnc_id, ret_key, ret_value ORDER BY hgnc_id, ret_value // Indicate the source of cell type information. +// APRIL 2024 - HRA changed "has_marker_component" to "characterized_by" UNION WITH GeneCUI -OPTIONAL MATCH (cGene:Code)<-[:CODE]-(pGene:Concept)-[:inverse_has_marker_component]->(pCL:Concept)-[:CODE]->(cCL:Code)-[rCL]->(tCL:Term) WHERE rCL.CUI=pCL.CUI AND pGene.CUI=GeneCUI AND cGene.SAB='HGNC' AND cCL.SAB='CL' RETURN DISTINCT toInteger(cGene.CODE) AS hgnc_id,'cell_types_source' as ret_key, cCL.CodeID + '|Human Reference Atlas' as ret_value +OPTIONAL MATCH (cGene:Code)<-[:CODE]-(pGene:Concept)-[:inverse_characterized_by]->(pCL:Concept)-[:CODE]->(cCL:Code)-[rCL]->(tCL:Term) WHERE rCL.CUI=pCL.CUI AND pGene.CUI=GeneCUI AND cGene.SAB='HGNC' AND cCL.SAB='CL' RETURN DISTINCT toInteger(cGene.CODE) AS hgnc_id,'cell_types_source' as ret_key, cCL.CodeID + '|Human Reference Atlas' as ret_value ORDER BY hgnc_id,cCL.CodeID + '|Human Reference Atlas' } +// APRIL 2024 bug fix check for null gene before calling fromlists + WITH hgnc_id, ret_key, COLLECT(ret_value) AS values -WITH hgnc_id,apoc.map.fromLists(COLLECT(ret_key),COLLECT(values)) AS map WHERE hgnc_id IS NOT NULL +WITH hgnc_id,apoc.map.fromLists(COLLECT(ret_key),COLLECT(values)) AS map RETURN hgnc_id, map['approved_symbol'] AS approved_symbol, map['approved_name'] AS approved_name, From d65918ba88d6a6d275bb1e5e87b1b1521200d3c1 Mon Sep 17 00:00:00 2001 From: AlanSimmons Date: Tue, 9 Apr 2024 15:39:07 -0500 Subject: [PATCH 17/22] Bug fixes in celltypedetail.cypher: 1. HRA changed label for "has_marker_component" to "characterized_by" 2. Converted initial subquery from using CODE to CodeID to allow for CL Codes with leading zeroes. --- src/hs_ontology_api/cypher/celltypedetail.cypher | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/hs_ontology_api/cypher/celltypedetail.cypher b/src/hs_ontology_api/cypher/celltypedetail.cypher index d9b5c95..b9ed0a8 100644 --- a/src/hs_ontology_api/cypher/celltypedetail.cypher +++ b/src/hs_ontology_api/cypher/celltypedetail.cypher @@ -12,8 +12,9 @@ CALL // The calling function in neo4j_logic.py will replace $ids. WITH [$ids] AS ids -OPTIONAL MATCH (pCL:Concept)-[:CODE]->(cCL:Code) WHERE cCL.SAB='CL' AND CASE WHEN ids[0]<>'' THEN ANY(id in ids WHERE cCL.CODE=id) ELSE 1=1 END RETURN DISTINCT pCL.CUI AS CLCUI -} +// APRIL 2024 Bug fix to use CodeID instead of CODE for cases of leading zeroes in strings. +OPTIONAL MATCH (pCL:Concept)-[:CODE]->(cCL:Code) +WHERE CASE WHEN ids[0]<>'' THEN ANY(id in ids WHERE cCL.CodeID='CL:'+id) ELSE 1=1 END RETURN DISTINCT pCL.CUI AS CLCUI} CALL { @@ -54,10 +55,11 @@ ORDER BY CLID UNION //CL-HGNC mappings via HRA +// APRIL 2024 - HRA changed "has_marker_component" to "characterized_by" //HGNC ID WITH CLCUI -OPTIONAL MATCH (cCL:Code)<-[:CODE]-(pCL:Concept)-[:has_marker_component]->(pGene:Concept)-[:CODE]->(cGene:Code)-[r]->(tGene:Term) +OPTIONAL MATCH (cCL:Code)<-[:CODE]-(pCL:Concept)-[:characterized_by]->(pGene:Concept)-[:CODE]->(cGene:Code)-[r]->(tGene:Term) WHERE pCL.CUI=CLCUI AND cGene.SAB='HGNC' AND r.CUI=pGene.CUI AND cCL.SAB='CL' AND type(r) IN ['ACR','PT'] WITH COLLECT(tGene.name) AS tgene_names, cGene.CodeID AS cgene_codeid, cCL.CodeID AS ccl_codeid WITH distinct ccl_codeid AS CLID, 'cell_types_genes' AS ret_key, cgene_codeid+'|'+apoc.text.join(tgene_names,'|') AS ret_value From edc772f6719f5ca6fce6f9460089351a8abb1637 Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Fri, 12 Apr 2024 14:33:00 -0400 Subject: [PATCH 18/22] Bump version to 2.0.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ac9f79c..227cea2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.10 +2.0.0 From 4197afb65ab2cbdd561c859f1a24a9f9b94f5a4d Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Fri, 12 Apr 2024 14:33:23 -0400 Subject: [PATCH 19/22] Update hs-ontology-api-spec.yaml version to 2.0.0 --- hs-ontology-api-spec.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hs-ontology-api-spec.yaml b/hs-ontology-api-spec.yaml index f01d91f..34ba3ec 100644 --- a/hs-ontology-api-spec.yaml +++ b/hs-ontology-api-spec.yaml @@ -2,7 +2,7 @@ openapi: 3.0.3 info: title: HubMAP/SenNet Ontology API (hs-ontology-api) description: The HuBMAP/SenNet Ontology API contains endpoints for querying a [UBKG](https://ubkg.docs.xconsortia.org/) instance with content from the [HuBMAP/SenNet context](https://ubkg.docs.xconsortia.org/contexts/#hubmapsennet-context). The hs-ontology-api imports the [ubkg-api](https://smart-api.info/ui/96e5b5c0b0efeef5b93ea98ac2794837), which encapsulates both basic connectivity to a UBKG instance and generic endpoint code. - version: 1.4.4 + version: 2.0.0 contact: name: GitHub repository url: https://github.com/x-atlas-consortia/hs-ontology-api @@ -1876,4 +1876,4 @@ components: schema: type: string description: name of schema - example: imc3d \ No newline at end of file + example: imc3d From cc52031c348b4db8cafb7d28e22075e8ecf3ca12 Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Fri, 12 Apr 2024 14:58:29 -0400 Subject: [PATCH 20/22] Increase to 4 workers and enable multi-threading --- src/main.py | 1 + src/uwsgi.ini | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main.py b/src/main.py index eb6c59a..6670d07 100755 --- a/src/main.py +++ b/src/main.py @@ -39,6 +39,7 @@ def make_flask_config(): app = UbkgAPI(make_flask_config(), Path(__file__).absolute().parent.parent).app + app.register_blueprint(assaytype_blueprint) app.register_blueprint(assayname_blueprint) app.register_blueprint(datasets_blueprint) diff --git a/src/uwsgi.ini b/src/uwsgi.ini index 1a3ca51..15854af 100644 --- a/src/uwsgi.ini +++ b/src/uwsgi.ini @@ -9,9 +9,12 @@ module = wsgi:application # Send logs to stdout instead of file so docker picks it up and writes to AWS CloudWatch log-master=true -# Master with 2 worker process (based on CPU number) +# Master with 4 worker process (based on CPU number) master = true -processes = 2 +processes = 4 + +# Enable multithreading +enable-threads = true # Use http socket for integration with nginx running on the same machine socket = localhost:5000 From 98357b0afb64f89d65bd0dfb37f6feca42c335af Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Fri, 12 Apr 2024 15:17:46 -0400 Subject: [PATCH 21/22] Update requirements.txt to use ubkg-api 2.1.1 --- src/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/requirements.txt b/src/requirements.txt index 08e596a..08f196a 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,5 +1,4 @@ -# ubkg-api==2.0.0 -git+https://github.com/x-atlas-consortia/ubkg-api.git@neo4jv5 +ubkg-api==2.1.0 Flask==2.1.3 neo4j==5.15.0 From 80bd72232a83f31e1fcea431e80609f084ba9b7c Mon Sep 17 00:00:00 2001 From: yuanzhou Date: Fri, 12 Apr 2024 15:20:05 -0400 Subject: [PATCH 22/22] Install ubkg-api2.1.1 --- src/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/requirements.txt b/src/requirements.txt index 08f196a..d013d6e 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,4 +1,4 @@ -ubkg-api==2.1.0 +ubkg-api==2.1.1 Flask==2.1.3 neo4j==5.15.0