feat: refactor ancestry mapping to include distance from descendant n…

…ode + implement functions to support curated list term mapping (#96)
chanzuckerberg · Mar 8, 2024 · 7fc3562 · 7fc3562
1 parent dfe0b39
commit 7fc3562
Show file tree

Hide file tree

Showing 21 changed files with 285 additions and 72 deletions.
diff --git a/api/python/src/cellxgene_ontology_guide/ontology_parser.py b/api/python/src/cellxgene_ontology_guide/ontology_parser.py
@@ -41,8 +41,8 @@ def _parse_ontology_name(self, term_id: str) -> str:
 
     def get_term_ancestors(self, term_id: str, include_self: bool = False) -> List[str]:
         """
-        Get the ancestor ontology terms for a given term. If include_self is True, the term itself will be included as an
-         ancestor.
+        Get the ancestor ontology terms for a given term. If include_self is True, the term itself will be included as
+        an ancestor.
 
          Example: get_term_ancestors("CL:0000005") -> ["CL:0000000", ...]
 
@@ -51,13 +51,30 @@ def get_term_ancestors(self, term_id: str, include_self: bool = False) -> List[s
         :return: flattened List[str] of ancestor terms
         """
         ontology_name = self._parse_ontology_name(term_id)
-        ancestors: List[str] = self.cxg_schema.ontology(ontology_name)[term_id]["ancestors"]
+        ancestors = list(self.cxg_schema.ontology(ontology_name)[term_id]["ancestors"].keys())
         return ancestors + [term_id] if include_self else ancestors
 
-    def get_term_list_ancestors(self, term_ids: str, include_self: bool = False) -> Dict[str, List[str]]:
+    def get_term_ancestors_with_distances(self, term_id: str, include_self: bool = False) -> Dict[str, int]:
         """
-        Get the ancestor ontology terms for each term in a list. If include_self is True, the term itself will be included
-         as an ancestor.
+        Get the ancestor ontology terms for a given term, and their distance from the term_id. If include_self is True,
+        the term itself will be included as an ancestor.
+
+         Example: get_term_ancestors_with_distances("CL:0000005") -> {"CL:0000000": 1, ...}
+
+        :param term_id: str ontology term to find ancestors for
+        :param include_self: boolean flag to include the term itself as an ancestor
+        :return: Dict[str, int] map of ancestor terms and their respective distances from the term_id
+        """
+        ontology_name = self._parse_ontology_name(term_id)
+        ancestors: Dict[str, int] = self.cxg_schema.ontology(ontology_name)[term_id]["ancestors"]
+        if include_self:
+            ancestors[term_id] = 0
+        return ancestors
+
+    def get_term_list_ancestors(self, term_ids: List[str], include_self: bool = False) -> Dict[str, List[str]]:
+        """
+        Get the ancestor ontology terms for each term in a list. If include_self is True, the term itself will be
+        included as an ancestor.
 
          Example: get_term_list_ancestors(["CL:0000003", "CL:0000005"], include_self=True) -> {
             "CL:0000003": ["CL:0000003"],
@@ -71,10 +88,106 @@ def get_term_list_ancestors(self, term_ids: str, include_self: bool = False) ->
         """
         return {term_id: self.get_term_ancestors(term_id, include_self) for term_id in term_ids}
 
+    def map_high_level_terms(self, term_ids: List[str], high_level_terms: List[str]) -> Dict[str, List[str]]:
+        """
+        Given a list of ontology term IDs and a list of high_level_terms to map them to, returns a dictionary with
+        format
+
+        {"CL:0000003": ["CL:0000000", ...], "CL:0000005": ["CL:0000000", ...]}
+
+        Where each term_id is mapped to a List[str] of high-level terms that it is a descendant of. Includes self
+        as a descendant.
+
+        :param term_ids: list of str ontology terms to map high level terms for
+        :param high_level_terms: list of str ontology terms that can be mapped to descendant term_ids
+        :return: Dictionary mapping str term IDs to their respective List[str] of ancestor terms from the input list.
+        Each key maps to empty list if there are no ancestors among the provided input.
+        """
+        ancestors = self.get_term_list_ancestors(term_ids, include_self=True)
+        for term_id in term_ids:
+            ancestors[term_id] = [
+                high_level_term for high_level_term in ancestors[term_id] if high_level_term in high_level_terms
+            ]
+        return ancestors
+
+    def get_distance_between_terms(self, term_id_1: str, term_id_2: str) -> int:
+        """
+        Get the distance between two ontology terms. The distance is defined as the number of edges between the
+        two terms. Terms must be from the same ontology. Returns -1 if terms are disjoint.
+
+        :param term_id_1: str ontology term to find distance for
+        :param term_id_2: str ontology term to find distance for
+        :return: int distance between the two terms, measured in number of edges between their shortest path.
+        """
+        lcas = self.get_lowest_common_ancestors(term_id_1, term_id_2)
+        if not lcas:
+            return -1
+        return int(
+            self.get_term_ancestors_with_distances(term_id_1, include_self=True)[lcas[0]]
+            + self.get_term_ancestors_with_distances(term_id_2, include_self=True)[lcas[0]]
+        )
+
+    def get_lowest_common_ancestors(self, term_id_1: str, term_id_2: str) -> List[str]:
+        """
+        Get the lowest common ancestors between two ontology terms that is from the given ontology.
+        Terms must be from the same ontology. Ontologies are DAGs, so there may be multiple lowest common ancestors.
+
+        :param term_id_1: str ontology term to find LCA for
+        :param term_id_2: str ontology term to find LCA for
+        :return: str term ID of the lowest common ancestor term
+        """
+        # include path to term itself
+        ontology = self._parse_ontology_name(term_id_1)
+        if ontology != self._parse_ontology_name(term_id_2):
+            return []
+        ancestors_1 = self.get_term_ancestors_with_distances(term_id_1, include_self=True)
+        ancestors_2 = self.get_term_ancestors_with_distances(term_id_2, include_self=True)
+        common_ancestors = set(ancestors_1.keys()) & set(ancestors_2.keys())
+        min_sum_distances = float("inf")
+        for ancestors in common_ancestors:
+            sum_distances = ancestors_1[ancestors] + ancestors_2[ancestors]
+            if sum_distances < min_sum_distances:
+                min_sum_distances = sum_distances
+        return [
+            ancestor
+            for ancestor in common_ancestors
+            if ancestors_1[ancestor] + ancestors_2[ancestor] == min_sum_distances
+        ]
+
+    def map_highest_level_term(self, term_ids: List[str], high_level_terms: List[str]) -> Dict[str, Union[str, None]]:
+        """
+        Given a list of ontology term IDs and a list of high_level_terms to map them to, returns a dictionary with
+        format
+
+        {"CL:0000003": "CL:0000000", "CL:0000005": "CL:0000000"}
+
+        Where each term_id is mapped to the highest level term that it is a descendant of, from the list provided. Includes
+        term itself as a descendant. Maps to None if term_id does not map to any high level terms among the provided input.
+
+        :param term_ids: list of str ontology terms to map high level terms for
+        :param high_level_terms: list of str ontology terms that can be mapped to descendant term_ids
+        :return: Dictionary mapping str term IDs to their respective List[str] of ancestor terms from the input list.
+        Each key maps to empty list if there are no ancestors among the provided input.
+        """
+        high_level_term_map = self.map_high_level_terms(term_ids, high_level_terms)
+        highest_level_term_map = dict()
+        for term_id in term_ids:
+            term_ancestors_and_distances = self.get_term_ancestors_with_distances(term_id, include_self=True)
+            # map term_id to the high_level_term with the longest distance from term_id
+            highest_level_term_map[term_id] = (
+                max(
+                    high_level_term_map[term_id],
+                    key=lambda high_level_term: term_ancestors_and_distances[high_level_term],
+                )
+                if high_level_term_map[term_id]
+                else None
+            )
+        return highest_level_term_map
+
     def get_terms_descendants(self, term_ids: List[str], include_self: bool = False) -> Dict[str, List[str]]:
         """
-        Get the descendant ontology terms for each term in a list. If include_self is True, the term itself will be included
-         as a descendant.
+        Get the descendant ontology terms for each term in a list. If include_self is True, the term itself will be
+         included as a descendant.
 
         Example: get_terms_descendants(["CL:0000003", "CL:0000005"], include_self=True) -> {
             "CL:0000003": ["CL:0000003", "CL:0000004", ...],
@@ -83,8 +196,8 @@ def get_terms_descendants(self, term_ids: List[str], include_self: bool = False)
 
         :param term_ids: list of str ontology terms to find descendants for
         :param include_self: boolean flag to include the term itself as an descendant
-        :return: Dictionary mapping str term IDs to their respective flattened List[str] of descendant terms. Maps to empty
-        list if there are no descendants.
+        :return: Dictionary mapping str term IDs to their respective flattened List[str] of descendant terms. Maps to
+        empty list if there are no descendants.
         """
         descendants_dict = dict()
         ontology_names = set()
@@ -96,7 +209,8 @@ def get_terms_descendants(self, term_ids: List[str], include_self: bool = False)
         for ontology in ontology_names:
             for candidate_descendant, candidate_metadata in self.cxg_schema.ontology(ontology).items():
                 for ancestor_id in descendants_dict:
-                    if ancestor_id in candidate_metadata["ancestors"]:
+                    ancestors = candidate_metadata["ancestors"].keys()
+                    if ancestor_id in ancestors:
                         descendants_dict[ancestor_id].append(candidate_descendant)
 
         return descendants_dict

diff --git a/api/python/tests/test_ontology_parser.py b/api/python/tests/test_ontology_parser.py
@@ -9,23 +9,35 @@
 @pytest.fixture
 def ontology_dict():
     return {
-        "CL:0000000": {"ancestors": [], "label": "cell A", "deprecated": False},
+        "CL:0000000": {"ancestors": {}, "label": "cell A", "deprecated": False},
         "CL:0000001": {
-            "ancestors": ["CL:0000000"],
+            "ancestors": {"CL:0000000": 1},
             "label": "cell B",
             "deprecated": False,
             "consider": ["CL:0000004"],
         },
-        "CL:0000002": {"ancestors": ["CL:0000000"], "label": "cell C", "deprecated": False},
+        "CL:0000002": {"ancestors": {"CL:0000000": 1}, "label": "cell C", "deprecated": False},
         "CL:0000003": {
-            "ancestors": ["CL:0000000"],
+            "ancestors": {"CL:0000000": 1},
             "label": "obsolete cell",
             "deprecated": True,
             "replaced_by": "CL:0000004",
             "comments": ["this term was deprecated in favor of a descendant term of CL:0000001"],
             "term_tracker": "http://example.com/issue/1234",
         },
-        "CL:0000004": {"ancestors": ["CL:0000001", "CL:0000000"], "label": "cell B2", "deprecated": False},
+        "CL:0000004": {
+            "ancestors": {"CL:0000000": 2, "CL:0000001": 1, "CL:0000002": 1},
+            "label": "cell BC",
+            "deprecated": False,
+        },
+        "CL:0000005": {
+            "ancestors": {"CL:0000000": 2, "CL:0000001": 1, "CL:0000002": 1},
+            "label": "cell BC2",
+            "deprecated": False,
+        },
+        "CL:0000006": {"ancestors": {"CL:0000000": 2, "CL:0000001": 1}, "label": "cell B2", "deprecated": False},
+        "CL:0000007": {"ancestors": {"CL:0000000": 2, "CL:0000001": 1}, "label": "cell B3", "deprecated": False},
+        "CL:0000008": {"ancestors": {}, "label": "cell unrelated", "deprecated": False},
     }
 
 
@@ -62,32 +74,64 @@ def test_parse_ontology_name__not_supported(ontology_parser):
 
 
 def test_get_term_ancestors(ontology_parser):
-    assert ontology_parser.get_term_ancestors("CL:0000004") == ["CL:0000001", "CL:0000000"]
+    assert ontology_parser.get_term_ancestors("CL:0000004") == ["CL:0000000", "CL:0000001", "CL:0000002"]
     assert ontology_parser.get_term_ancestors("CL:0000004", include_self=True) == [
-        "CL:0000001",
         "CL:0000000",
+        "CL:0000001",
+        "CL:0000002",
         "CL:0000004",
     ]
 
 
+def test_get_term_ancestors_with_distances(ontology_parser):
+    assert ontology_parser.get_term_ancestors_with_distances("CL:0000004") == {
+        "CL:0000000": 2,
+        "CL:0000001": 1,
+        "CL:0000002": 1,
+    }
+    assert ontology_parser.get_term_ancestors_with_distances("CL:0000004", include_self=True) == {
+        "CL:0000000": 2,
+        "CL:0000001": 1,
+        "CL:0000002": 1,
+        "CL:0000004": 0,
+    }
+
+
 def test_get_term_list_ancestors(ontology_parser):
     assert ontology_parser.get_term_list_ancestors(["CL:0000000", "CL:0000004"]) == {
         "CL:0000000": [],
-        "CL:0000004": ["CL:0000001", "CL:0000000"],
+        "CL:0000004": ["CL:0000000", "CL:0000001", "CL:0000002"],
     }
     assert ontology_parser.get_term_list_ancestors(["CL:0000000", "CL:0000004"], include_self=True) == {
         "CL:0000000": ["CL:0000000"],
-        "CL:0000004": ["CL:0000001", "CL:0000000", "CL:0000004"],
+        "CL:0000004": ["CL:0000000", "CL:0000001", "CL:0000002", "CL:0000004"],
     }
 
 
 def test_get_terms_descendants(ontology_parser):
     assert ontology_parser.get_terms_descendants(["CL:0000000", "CL:0000004"]) == {
-        "CL:0000000": ["CL:0000001", "CL:0000002", "CL:0000003", "CL:0000004"],
+        "CL:0000000": [
+            "CL:0000001",
+            "CL:0000002",
+            "CL:0000003",
+            "CL:0000004",
+            "CL:0000005",
+            "CL:0000006",
+            "CL:0000007",
+        ],
         "CL:0000004": [],
     }
     assert ontology_parser.get_terms_descendants(["CL:0000000", "CL:0000004"], include_self=True) == {
-        "CL:0000000": ["CL:0000000", "CL:0000001", "CL:0000002", "CL:0000003", "CL:0000004"],
+        "CL:0000000": [
+            "CL:0000000",
+            "CL:0000001",
+            "CL:0000002",
+            "CL:0000003",
+            "CL:0000004",
+            "CL:0000005",
+            "CL:0000006",
+            "CL:0000007",
+        ],
         "CL:0000004": ["CL:0000004"],
     }
 
@@ -116,7 +160,55 @@ def test_get_term_metadata(ontology_parser):
 
 
 def test_get_term_label(ontology_parser):
-    assert ontology_parser.get_term_label("CL:0000004") == "cell B2"
+    assert ontology_parser.get_term_label("CL:0000004") == "cell BC"
+
+
+def test_map_high_level_terms(ontology_parser):
+    assert ontology_parser.map_high_level_terms(
+        term_ids=["CL:0000000", "CL:0000008", "CL:0000004"],
+        high_level_terms=["CL:0000000", "CL:0000001"],
+    ) == {"CL:0000000": ["CL:0000000"], "CL:0000008": [], "CL:0000004": ["CL:0000000", "CL:0000001"]}
+
+
+def test_map_highest_level_term(ontology_parser):
+    assert ontology_parser.map_highest_level_term(
+        term_ids=["CL:0000000", "CL:0000008", "CL:0000004"],
+        high_level_terms=["CL:0000000", "CL:0000001"],
+    ) == {"CL:0000000": "CL:0000000", "CL:0000008": None, "CL:0000004": "CL:0000000"}
+
+
+def test_get_lowest_common_ancestors(ontology_parser):
+    # root node LCA
+    assert ontology_parser.get_lowest_common_ancestors(term_id_1="CL:0000003", term_id_2="CL:0000005") == ["CL:0000000"]
+
+    # sibling LCA
+    assert ontology_parser.get_lowest_common_ancestors(term_id_1="CL:0000006", term_id_2="CL:0000007") == ["CL:0000001"]
+
+    # parent-child LCA
+    assert ontology_parser.get_lowest_common_ancestors(term_id_1="CL:0000002", term_id_2="CL:0000005") == ["CL:0000002"]
+
+    # multiple node
+    lcas = ontology_parser.get_lowest_common_ancestors(term_id_1="CL:0000004", term_id_2="CL:0000005")
+    assert len(lcas) == 2
+    assert "CL:0000001" in lcas
+    assert "CL:0000002" in lcas
+
+    # disjoint
+    assert ontology_parser.get_lowest_common_ancestors(term_id_1="CL:0000001", term_id_2="CL:0000008") == []
+
+
+def test_get_distance_between_terms(ontology_parser):
+    # distance when root node is lca
+    assert ontology_parser.get_distance_between_terms(term_id_1="CL:0000003", term_id_2="CL:0000005") == 3
+
+    # parent-child distance
+    assert ontology_parser.get_distance_between_terms(term_id_1="CL:0000002", term_id_2="CL:0000005") == 1
+
+    # multiple LCAs distance
+    assert ontology_parser.get_distance_between_terms(term_id_1="CL:0000004", term_id_2="CL:0000005") == 2
+
+    # disjoint distance
+    assert ontology_parser.get_distance_between_terms(term_id_1="CL:0000001", term_id_2="CL:0000008") == -1
 
 
 def test_get_ontology_download_url(ontology_parser):

diff --git a/artifact-schemas/all_ontology_schema.json b/artifact-schemas/all_ontology_schema.json
@@ -1,10 +1,10 @@
 {
-  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$schema": "http://json-schema.org/draft-07,/schema#",
   "title": "Valid Ontology Term JSON Schema",
   "description": "Schema for file containing metadata for Ontology Terms accepted in dataset submissions to CZ CellXGene Data Portal.",
   "type": "object",
   "patternProperties": {
-    "^(EFO|UBERON|CL|HANCESTRO|HsapDv|MmusDv|PATO|NCBITaxon|MONDO):[0-9]{7}$": {
+    "^(EFO|UBERON|CL|HANCESTRO|HsapDv|MmusDv|PATO|NCBITaxon|MONDO):[0-9]{7,}$": {
       "type": "object",
       "properties": {
         "label": {
@@ -16,13 +16,13 @@
           "description": "Indicates whether the ontology entry is deprecated."
         },
         "ancestors": {
-          "type": "array",
-          "items": {
-            "type": "string",
-            "pattern": "^(EFO|UBERON|CL|HANCESTRO|HsapDv|MmusDv|PATO|NCBITaxon|MONDO):[0-9]{7}$",
-            "description": "List of ancestor IDs for the ontology entry."
-          },
-          "description": "An array of ancestor ontology terms that this term is a subclass of."
+          "type": "object",
+          "description": "A map of ancestor ontology terms that this term is a subclass of, keyed to the distance from the term.",
+          "patternProperties": {
+            "^(EFO|UBERON|CL|HANCESTRO|HsapDv|MmusDv|PATO|NCBITaxon|MONDO):[0-9]{7,}$": {
+              "type": "integer"
+            }
+          }
         },
         "comments": {
           "type": "array",
@@ -47,7 +47,7 @@
         },
         "replaced_by": {
           "type": "string",
-          "pattern": "^(EFO|UBERON|CL|HANCESTRO|HsapDv|MmusDv|PATO|NCBITaxon|MONDO):[0-9]{7}$",
+          "pattern": "^(EFO|UBERON|CL|HANCESTRO|HsapDv|MmusDv|PATO|NCBITaxon|MONDO):[0-9]{7,}$",
           "description": "If deprecated, the ID of the ontology entry that should canonically replace this one."
         }
       },

diff --git a/artifact-schemas/cell_class_list_schema.json b/artifact-schemas/cell_class_list_schema.json
@@ -5,7 +5,7 @@
   "type": "array",
   "items": {
     "type": "string",
-    "pattern": "^CL:[0-9]{7}$"
+    "pattern": "^CL:[0-9]{7,}$"
   },
   "minItems": 1,
   "uniqueItems": true