Replace littletable with redis-om(redisearch) (#1788)

* replace little table with redis-om

* add partial word search

https://redis.io/docs/interact/search-and-query/advanced-concepts/query_syntax/#prefix-matching

* limit prefix expansion to 3 words

* use redis-stack inplace of redis image

* handle exceptions raised due to cache miss

* add health check for containers

* fix page size issue

* fix test

* escape tokens

* add is leaf filter for icd11 items

* fix bucket auth

* fix localstack cors

* use a different endpoint for docker health check

* only load icd11 in cache which has code

* fix imports

* fix key error

* revert docker changes

* fix ping endpoint

* fix icd11 search by code

* fix not found error handling

* make icd tests cover more cases

* fix ping endpoint

.dockerignore

@@ -3,3 +3,4 @@
 htmlcov
 staticfiles
 .coverage
+care/media/

.github/workflows/test-base.yml

@@ -16,7 +16,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ hashFiles('Pipfile.lock', 'docker/prod.Dockerfile') }}
+          key: ${{ runner.os }}-buildx-${{ hashFiles('Pipfile.lock', 'docker/dev.Dockerfile') }}
           restore-keys: |
             ${{ runner.os }}-buildx-
 
@@ -30,7 +30,7 @@ jobs:
           files: docker-compose.yaml,docker-compose.local.yaml
 
       - name: Start services
-        run: docker compose -f docker-compose.yaml -f docker-compose.local.yaml up -d --no-build
+        run: docker compose -f docker-compose.yaml -f docker-compose.local.yaml up -d --wait --no-build
 
       - name: Check migrations
         run: make checkmigration

Makefile

@@ -20,7 +20,7 @@ build:
 	docker compose -f docker-compose.yaml -f $(docker_config_file) build
 
 up:
-	docker compose -f docker-compose.yaml -f $(docker_config_file) up -d
+	docker compose -f docker-compose.yaml -f $(docker_config_file) up -d --wait
 
 down:
 	docker compose -f docker-compose.yaml -f $(docker_config_file) down
@@ -34,16 +34,16 @@ list:
 logs:
 	docker compose -f docker-compose.yaml -f $(docker_config_file) logs
 
-checkmigration: up
+checkmigration:
 	docker compose exec backend bash -c "python manage.py makemigrations --check --dry-run"
 
-makemigrations: up
+makemigrations:
 	docker compose exec backend bash -c "python manage.py makemigrations"
 
-test: up
+test:
 	docker compose exec backend bash -c "python manage.py test --keepdb --parallel"
 
-test-coverage: up
+test-coverage:
 	docker compose exec backend bash -c "coverage run manage.py test --settings=config.settings.test --keepdb --parallel"
 	docker compose exec backend bash -c "coverage combine || true; coverage xml"
 	docker compose cp backend:/app/coverage.xml coverage.xml

Pipfile

@@ -32,7 +32,6 @@ gunicorn = "==21.2.0"
 healthy-django = "==0.1.0"
 jsonschema = "==4.20.0"
 jwcrypto = "==1.5.1"
-littletable = "==2.2.3"
 newrelic = "==9.3.0"
 pillow = "==10.1.0"
 psycopg = "==3.1.14"
@@ -41,10 +40,11 @@ pydantic = "==1.10.12" # fix for fhir.resources < 7.0.2
 pyjwt = "==2.8.0"
 python-slugify = "==8.0.1"
 pywebpush = "==1.14.0"
-redis = {extras = ["hiredis"], version = "==5.0.0"}
+redis = {extras = ["hiredis"], version = "<5.0.0"} # constraint for redis-om
 requests = "==2.31.0"
 sentry-sdk = "==1.30.0"
 whitenoise = "==6.5.0"
+redis-om = "==0.2.1"
 
 [dev-packages]
 black = "==23.9.1"

care/abdm/utils/fhir.py

@@ -34,6 +34,7 @@
 from care.facility.models.file_upload import FileUpload
 from care.facility.models.icd11_diagnosis import REVERSE_CONDITION_VERIFICATION_STATUSES
 from care.facility.models.patient_investigation import InvestigationValue
+from care.facility.static_data.icd11 import get_icd11_diagnosis_object_by_id
 
 
 class Fhir:
@@ -137,8 +138,6 @@ def _organization(self):
         return self._organization_profile
 
     def _condition(self, diagnosis_id, verification_status):
-        from care.facility.static_data.icd11 import get_icd11_diagnosis_object_by_id
-
         diagnosis = get_icd11_diagnosis_object_by_id(diagnosis_id)
         [code, label] = diagnosis.label.split(" ", 1)
         condition_profile = Condition(

care/facility/api/serializers/consultation_diagnosis.py

@@ -7,6 +7,7 @@
     ConsultationDiagnosis,
 )
 from care.facility.models.icd11_diagnosis import ICD11Diagnosis
+from care.facility.static_data.icd11 import get_icd11_diagnosis_object_by_id
 from care.users.api.serializers.user import UserBaseMinimumSerializer
 
 
@@ -30,8 +31,6 @@ class ConsultationDiagnosisSerializer(serializers.ModelSerializer):
     created_by = UserBaseMinimumSerializer(read_only=True)
 
     def get_diagnosis_object(self, obj):
-        from care.facility.static_data.icd11 import get_icd11_diagnosis_object_by_id
-
         return get_icd11_diagnosis_object_by_id(obj.diagnosis_id, as_dict=True)
 
     class Meta: