Skip to content

Commit

Permalink
Refactor ICD11 Table; Cleanup in-memory search (#1636)
Browse files Browse the repository at this point in the history
* Refactor ICD11 Table

* optimize memory usage

* gracefully handle relation not existing

* check table exists before loading from db

* Apply suggestions from code review

Co-authored-by: Aakash Singh <[email protected]>

* move fetch_data to commands

* fix id type in in-memory

* patch

* merge migrations

---------

Co-authored-by: Aakash Singh <[email protected]>
  • Loading branch information
rithviknishad and sainak authored Oct 5, 2023
1 parent 0a4a4c4 commit fd7fd29
Show file tree
Hide file tree
Showing 7 changed files with 177 additions and 59 deletions.
Original file line number Diff line number Diff line change
@@ -1,17 +1,46 @@
import json

from django.core.management import BaseCommand, CommandError

from care.facility.models.meta_icd11_diagnosis import MetaICD11Diagnosis
from care.facility.static_data.icd11 import fetch_data
from care.facility.models.icd11_diagnosis import ICD11Diagnosis


def fetch_data():
with open("data/icd11.json", "r") as json_file:
return json.load(json_file)


ICD11_ID_SUFFIX_TO_INT = {
"mms": 1,
"other": 2,
"unspecified": 3,
}


def icd11_id_to_int(icd11_id):
"""
Maps ICD11 ID to an integer.
Eg:
- http://id.who.int/icd/entity/594985340 -> 594985340
- http://id.who.int/icd/entity/594985340/mms -> 5949853400001
- http://id.who.int/icd/entity/594985340/mms/unspecified -> 5949853400003
"""
entity_id = icd11_id.replace("http://id.who.int/icd/entity/", "")
if entity_id.isnumeric():
return int(entity_id)
segments = entity_id.split("/")
return int(segments[0]) * 1e3 + ICD11_ID_SUFFIX_TO_INT[segments[-1]]


class Command(BaseCommand):
"""
Management command to load ICD11 diagnoses to database. Not for production
use.
Usage: python manage.py load_meta_icd11_diagnosis
Usage: python manage.py load_icd11_diagnoses
"""

help = "Loads ICD11 data to a table in to database."
help = "Loads ICD11 diagnoses data to database"

data = []
roots_lookup = {}
Expand All @@ -29,8 +58,9 @@ class Command(BaseCommand):
"""

CLASS_KIND_DB_KEYS = {
"block": "root_block",
"category": "root_category",
"chapter": "meta_chapter",
"block": "meta_root_block",
"category": "meta_root_category",
}

ICD11_GROUP_LABEL_PRETTY = {
Expand All @@ -55,13 +85,12 @@ class Command(BaseCommand):
"19 Certain conditions originating in the perinatal period": "Neonatology",
"20 Developmental anomalies": "Developmental Anomalies",
"21 Symptoms, signs or clinical findings, not elsewhere classified": "Others",
"22 Injury, poisoning or certain other consequences of external causes": "Injury, Poisoning ",
"22 Injury, poisoning or certain other consequences of external causes": "Injury, Poisoning",
"23 External causes of morbidity or mortality": "External Causes of Injury",
"24 Factors influencing health status or contact with health services": None,
"25 Codes for special purposes": "Codes for special purposes",
"26 Supplementary Chapter Traditional Medicine Conditions - Module I": None,
"V Supplementary section for functioning assessment": "Functioning assessment ",
"X Extension Codes": "NOT RELEVANT",
"V Supplementary section for functioning assessment": "Functioning assessment",
}

def find_roots(self, item):
Expand Down Expand Up @@ -98,7 +127,7 @@ def my(x):
)

def handle(self, *args, **options):
print("Loading ICD11 data to DB Table (meta_icd11_diagnosis)...")
print("Loading ICD11 diagnoses data to database...")
try:
self.data = fetch_data()

Expand All @@ -110,29 +139,27 @@ def roots(item):
result = {
self.CLASS_KIND_DB_KEYS.get(k, k): v for k, v in roots.items()
}
result["chapter_short"] = mapped
result["deleted"] = mapped is None
result["meta_chapter_short"] = mapped
result["meta_hidden"] = mapped is None
return result

MetaICD11Diagnosis.objects.all().delete()
MetaICD11Diagnosis.objects.bulk_create(
ICD11Diagnosis.objects.all().delete()
ICD11Diagnosis.objects.bulk_create(
[
MetaICD11Diagnosis(
id=icd11_object["ID"],
_id=int(icd11_object["ID"].split("/")[-1]),
average_depth=icd11_object["averageDepth"],
is_adopted_child=icd11_object["isAdoptedChild"],
parent_id=icd11_object["parentId"],
class_kind=icd11_object["classKind"],
is_leaf=icd11_object["isLeaf"],
label=icd11_object["label"],
breadth_value=icd11_object["breadthValue"],
**roots(icd11_object),
ICD11Diagnosis(
id=icd11_id_to_int(obj["ID"]),
icd11_id=obj["ID"],
label=obj["label"],
class_kind=obj["classKind"],
is_leaf=obj["isLeaf"],
parent_id=obj["parentId"] and icd11_id_to_int(obj["parentId"]),
average_depth=obj["averageDepth"],
is_adopted_child=obj["isAdoptedChild"],
breadth_value=obj["breadthValue"],
**roots(obj),
)
for icd11_object in self.data
if icd11_object["ID"].split("/")[-1].isnumeric()
for obj in self.data
]
)
print("Done loading ICD11 data to database.")
except Exception as e:
raise CommandError(e)
57 changes: 57 additions & 0 deletions care/facility/migrations/0388_icd11diagnosis.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
# Generated by Django 4.2.2 on 2023-09-25 13:00

import django.db.models.deletion
from django.core.management import call_command
from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("facility", "0387_merge_20230911_2303"),
]

operations = [
migrations.CreateModel(
name="ICD11Diagnosis",
fields=[
("id", models.BigIntegerField(primary_key=True, serialize=False)),
("icd11_id", models.CharField(max_length=255, unique=True)),
("label", models.CharField(max_length=255)),
(
"class_kind",
models.CharField(
choices=[
("chapter", "Chapter"),
("block", "Block"),
("category", "Category"),
],
max_length=255,
),
),
("is_leaf", models.BooleanField()),
("average_depth", models.IntegerField()),
("is_adopted_child", models.BooleanField()),
(
"breadth_value",
models.DecimalField(decimal_places=22, max_digits=24),
),
("meta_hidden", models.BooleanField(default=False)),
("meta_chapter", models.CharField(max_length=255)),
("meta_chapter_short", models.CharField(max_length=255, null=True)),
("meta_root_block", models.CharField(max_length=255, null=True)),
("meta_root_category", models.CharField(max_length=255, null=True)),
(
"parent",
models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.DO_NOTHING,
to="facility.icd11diagnosis",
),
),
],
),
migrations.RunPython(
lambda apps, schema_editor: call_command("load_icd11_diagnoses_data"),
reverse_code=migrations.RunPython.noop,
),
]
12 changes: 12 additions & 0 deletions care/facility/migrations/0389_merge_20231005_2247.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Generated by Django 4.2.5 on 2023-10-05 17:17

from django.db import migrations


class Migration(migrations.Migration):
dependencies = [
("facility", "0388_goal_goalentry_goalproperty_goalpropertyentry"),
("facility", "0388_icd11diagnosis"),
]

operations = []
1 change: 1 addition & 0 deletions care/facility/models/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from .bed import * # noqa
from .daily_round import * # noqa
from .facility import * # noqa
from .icd11_diagnosis import * # noqa
from .inventory import * # noqa
from .meta_icd11_diagnosis import * # noqa
from .patient import * # noqa
Expand Down
33 changes: 33 additions & 0 deletions care/facility/models/icd11_diagnosis.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
from django.db import models


class ICD11ClassKind(models.TextChoices):
CHAPTER = "chapter"
BLOCK = "block"
CATEGORY = "category"


class ICD11Diagnosis(models.Model):
"""
Use ICDDiseases for in-memory search.
"""

id = models.BigIntegerField(primary_key=True)
icd11_id = models.CharField(max_length=255, unique=True)
label = models.CharField(max_length=255)
class_kind = models.CharField(max_length=255, choices=ICD11ClassKind.choices)
is_leaf = models.BooleanField()
parent = models.ForeignKey("self", on_delete=models.DO_NOTHING, null=True)
average_depth = models.IntegerField()
is_adopted_child = models.BooleanField()
breadth_value = models.DecimalField(max_digits=24, decimal_places=22)

# Meta fields
meta_hidden = models.BooleanField(default=False)
meta_chapter = models.CharField(max_length=255)
meta_chapter_short = models.CharField(max_length=255, null=True)
meta_root_block = models.CharField(max_length=255, null=True)
meta_root_category = models.CharField(max_length=255, null=True)

def __str__(self) -> str:
return self.label
2 changes: 2 additions & 0 deletions care/facility/models/meta_icd11_diagnosis.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
class MetaICD11Diagnosis(models.Model):
"""
Not for production use. For Metabase purposes only. Do not build relations to this model.
Deprecated in favor of ICD11Diagnosis. This table will be removed in the future.
"""

id = models.CharField(max_length=255, primary_key=True)
Expand Down
48 changes: 17 additions & 31 deletions care/facility/static_data/icd11.py
Original file line number Diff line number Diff line change
@@ -1,43 +1,29 @@
import contextlib
import json

from django.db import connection
from littletable import Table

from care.facility.models.icd11_diagnosis import ICD11Diagnosis

def fetch_data():
with open("data/icd11.json", "r") as json_file:
return json.load(json_file)


def is_numeric(val):
if str(val).isnumeric():
return val
return -1
def fetch_from_db():
# This is a hack to prevent the migration from failing when the table does not exist
all_tables = connection.introspection.table_names()
if "facility_icd11diagnosis" in all_tables:
return [
{
"id": str(diagnosis["id"]),
"label": diagnosis["label"],
}
for diagnosis in ICD11Diagnosis.objects.filter(is_leaf=True).values(
"id", "label"
)
]
return []


ICDDiseases = Table("ICD11")
icd11_objects = fetch_data()
entity_id = ""
IGNORE_FIELDS = [
"isLeaf",
"classKind",
"isAdoptedChild",
"averageDepth",
"breadthValue",
"Suggested",
]

for icd11_object in icd11_objects:
for field in IGNORE_FIELDS:
icd11_object.pop(field, "")
icd11_object["id"] = icd11_object.pop("ID")
entity_id = icd11_object["id"].split("/")[-1]
icd11_object["id"] = is_numeric(entity_id)
if icd11_object["id"] == -1:
continue
if icd11_object["id"]:
ICDDiseases.insert(icd11_object)

ICDDiseases.insert_many(fetch_from_db())
ICDDiseases.create_search_index("label")
ICDDiseases.create_index("id", unique=True)

Expand Down

0 comments on commit fd7fd29

Please sign in to comment.