Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor ICD11 Table; Cleanup in-memory search #1636

Merged
merged 10 commits into from
Oct 5, 2023
Original file line number Diff line number Diff line change
@@ -1,17 +1,46 @@
import json

from django.core.management import BaseCommand, CommandError

from care.facility.models.meta_icd11_diagnosis import MetaICD11Diagnosis
from care.facility.static_data.icd11 import fetch_data
from care.facility.models.icd11_diagnosis import ICD11Diagnosis


def fetch_data():
with open("data/icd11.json", "r") as json_file:
return json.load(json_file)


ICD11_ID_SUFFIX_TO_INT = {
"mms": 1,
"other": 2,
"unspecified": 3,
}


def icd11_id_to_int(icd11_id):
"""
Maps ICD11 ID to an integer.

Eg:
- http://id.who.int/icd/entity/594985340 -> 594985340
- http://id.who.int/icd/entity/594985340/mms -> 5949853400001
- http://id.who.int/icd/entity/594985340/mms/unspecified -> 5949853400003
"""
entity_id = icd11_id.replace("http://id.who.int/icd/entity/", "")
if entity_id.isnumeric():
return int(entity_id)
segments = entity_id.split("/")
return int(segments[0]) * 1e3 + ICD11_ID_SUFFIX_TO_INT[segments[-1]]


class Command(BaseCommand):
"""
Management command to load ICD11 diagnoses to database. Not for production
use.
Usage: python manage.py load_meta_icd11_diagnosis
Usage: python manage.py load_icd11_diagnoses
"""

help = "Loads ICD11 data to a table in to database."
help = "Loads ICD11 diagnoses data to database"

data = []
roots_lookup = {}
Expand All @@ -29,8 +58,9 @@ class Command(BaseCommand):
"""

CLASS_KIND_DB_KEYS = {
"block": "root_block",
"category": "root_category",
"chapter": "meta_chapter",
"block": "meta_root_block",
"category": "meta_root_category",
}

ICD11_GROUP_LABEL_PRETTY = {
Expand All @@ -55,13 +85,12 @@ class Command(BaseCommand):
"19 Certain conditions originating in the perinatal period": "Neonatology",
"20 Developmental anomalies": "Developmental Anomalies",
"21 Symptoms, signs or clinical findings, not elsewhere classified": "Others",
"22 Injury, poisoning or certain other consequences of external causes": "Injury, Poisoning ",
"22 Injury, poisoning or certain other consequences of external causes": "Injury, Poisoning",
"23 External causes of morbidity or mortality": "External Causes of Injury",
"24 Factors influencing health status or contact with health services": None,
"25 Codes for special purposes": "Codes for special purposes",
"26 Supplementary Chapter Traditional Medicine Conditions - Module I": None,
"V Supplementary section for functioning assessment": "Functioning assessment ",
"X Extension Codes": "NOT RELEVANT",
"V Supplementary section for functioning assessment": "Functioning assessment",
}

def find_roots(self, item):
Expand Down Expand Up @@ -98,7 +127,7 @@ def my(x):
)

def handle(self, *args, **options):
print("Loading ICD11 data to DB Table (meta_icd11_diagnosis)...")
print("Loading ICD11 diagnoses data to database...")
try:
self.data = fetch_data()

Expand All @@ -110,29 +139,27 @@ def roots(item):
result = {
self.CLASS_KIND_DB_KEYS.get(k, k): v for k, v in roots.items()
}
result["chapter_short"] = mapped
result["deleted"] = mapped is None
result["meta_chapter_short"] = mapped
result["meta_hidden"] = mapped is None
return result

MetaICD11Diagnosis.objects.all().delete()
MetaICD11Diagnosis.objects.bulk_create(
ICD11Diagnosis.objects.all().delete()
ICD11Diagnosis.objects.bulk_create(
[
MetaICD11Diagnosis(
id=icd11_object["ID"],
_id=int(icd11_object["ID"].split("/")[-1]),
average_depth=icd11_object["averageDepth"],
is_adopted_child=icd11_object["isAdoptedChild"],
parent_id=icd11_object["parentId"],
class_kind=icd11_object["classKind"],
is_leaf=icd11_object["isLeaf"],
label=icd11_object["label"],
breadth_value=icd11_object["breadthValue"],
**roots(icd11_object),
ICD11Diagnosis(
id=icd11_id_to_int(obj["ID"]),
icd11_id=obj["ID"],
label=obj["label"],
class_kind=obj["classKind"],
is_leaf=obj["isLeaf"],
parent_id=obj["parentId"] and icd11_id_to_int(obj["parentId"]),
average_depth=obj["averageDepth"],
is_adopted_child=obj["isAdoptedChild"],
breadth_value=obj["breadthValue"],
**roots(obj),
)
for icd11_object in self.data
if icd11_object["ID"].split("/")[-1].isnumeric()
for obj in self.data
]
)
print("Done loading ICD11 data to database.")
except Exception as e:
raise CommandError(e)
57 changes: 57 additions & 0 deletions care/facility/migrations/0388_icd11diagnosis.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
# Generated by Django 4.2.2 on 2023-09-25 13:00

import django.db.models.deletion
from django.core.management import call_command
from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("facility", "0387_merge_20230911_2303"),
]

operations = [
migrations.CreateModel(
name="ICD11Diagnosis",
fields=[
("id", models.BigIntegerField(primary_key=True, serialize=False)),
("icd11_id", models.CharField(max_length=255, unique=True)),
("label", models.CharField(max_length=255)),
(
"class_kind",
models.CharField(
choices=[
("chapter", "Chapter"),
("block", "Block"),
("category", "Category"),
],
max_length=255,
),
),
("is_leaf", models.BooleanField()),
("average_depth", models.IntegerField()),
("is_adopted_child", models.BooleanField()),
(
"breadth_value",
models.DecimalField(decimal_places=22, max_digits=24),
),
("meta_hidden", models.BooleanField(default=False)),
("meta_chapter", models.CharField(max_length=255)),
("meta_chapter_short", models.CharField(max_length=255, null=True)),
("meta_root_block", models.CharField(max_length=255, null=True)),
("meta_root_category", models.CharField(max_length=255, null=True)),
(
"parent",
models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.DO_NOTHING,
to="facility.icd11diagnosis",
),
),
],
),
migrations.RunPython(
lambda apps, schema_editor: call_command("load_icd11_diagnoses_data"),
reverse_code=migrations.RunPython.noop,
),
]
12 changes: 12 additions & 0 deletions care/facility/migrations/0389_merge_20231005_2247.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Generated by Django 4.2.5 on 2023-10-05 17:17

from django.db import migrations


class Migration(migrations.Migration):
dependencies = [
("facility", "0388_goal_goalentry_goalproperty_goalpropertyentry"),
("facility", "0388_icd11diagnosis"),
]

operations = []
1 change: 1 addition & 0 deletions care/facility/models/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from .bed import * # noqa
from .daily_round import * # noqa
from .facility import * # noqa
from .icd11_diagnosis import * # noqa
from .inventory import * # noqa
from .meta_icd11_diagnosis import * # noqa
from .patient import * # noqa
Expand Down
33 changes: 33 additions & 0 deletions care/facility/models/icd11_diagnosis.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
from django.db import models


class ICD11ClassKind(models.TextChoices):
CHAPTER = "chapter"
BLOCK = "block"
CATEGORY = "category"


class ICD11Diagnosis(models.Model):
"""
Use ICDDiseases for in-memory search.
"""

id = models.BigIntegerField(primary_key=True)
icd11_id = models.CharField(max_length=255, unique=True)
label = models.CharField(max_length=255)
class_kind = models.CharField(max_length=255, choices=ICD11ClassKind.choices)
is_leaf = models.BooleanField()
parent = models.ForeignKey("self", on_delete=models.DO_NOTHING, null=True)
average_depth = models.IntegerField()
is_adopted_child = models.BooleanField()
breadth_value = models.DecimalField(max_digits=24, decimal_places=22)

# Meta fields
meta_hidden = models.BooleanField(default=False)
meta_chapter = models.CharField(max_length=255)
meta_chapter_short = models.CharField(max_length=255, null=True)
meta_root_block = models.CharField(max_length=255, null=True)
meta_root_category = models.CharField(max_length=255, null=True)

def __str__(self) -> str:
return self.label

Check warning on line 33 in care/facility/models/icd11_diagnosis.py

View check run for this annotation

Codecov / codecov/patch

care/facility/models/icd11_diagnosis.py#L33

Added line #L33 was not covered by tests
2 changes: 2 additions & 0 deletions care/facility/models/meta_icd11_diagnosis.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
class MetaICD11Diagnosis(models.Model):
"""
Not for production use. For Metabase purposes only. Do not build relations to this model.

Deprecated in favor of ICD11Diagnosis. This table will be removed in the future.
"""

id = models.CharField(max_length=255, primary_key=True)
Expand Down
48 changes: 17 additions & 31 deletions care/facility/static_data/icd11.py
Original file line number Diff line number Diff line change
@@ -1,43 +1,29 @@
import contextlib
import json

from django.db import connection
from littletable import Table

from care.facility.models.icd11_diagnosis import ICD11Diagnosis

def fetch_data():
with open("data/icd11.json", "r") as json_file:
return json.load(json_file)


def is_numeric(val):
if str(val).isnumeric():
return val
return -1
def fetch_from_db():
# This is a hack to prevent the migration from failing when the table does not exist
all_tables = connection.introspection.table_names()
if "facility_icd11diagnosis" in all_tables:
return [
{
"id": str(diagnosis["id"]),
"label": diagnosis["label"],
}
for diagnosis in ICD11Diagnosis.objects.filter(is_leaf=True).values(
"id", "label"
)
]
return []


ICDDiseases = Table("ICD11")
icd11_objects = fetch_data()
entity_id = ""
IGNORE_FIELDS = [
"isLeaf",
"classKind",
"isAdoptedChild",
"averageDepth",
"breadthValue",
"Suggested",
]

for icd11_object in icd11_objects:
for field in IGNORE_FIELDS:
icd11_object.pop(field, "")
icd11_object["id"] = icd11_object.pop("ID")
entity_id = icd11_object["id"].split("/")[-1]
icd11_object["id"] = is_numeric(entity_id)
if icd11_object["id"] == -1:
continue
if icd11_object["id"]:
ICDDiseases.insert(icd11_object)

ICDDiseases.insert_many(fetch_from_db())
ICDDiseases.create_search_index("label")
ICDDiseases.create_index("id", unique=True)

Expand Down
Loading