feat: add Meilisearch-compatible search engine with tests (#164)

The goal of this change is to introduce a search engine that is compatible with the edx-search API but that uses Meilisearch instead of Elasticsearch. That way, we can replace one by the other across edx-platform by simply changing a single SEARCH_ENGINE django setting. There are a couple of differences between Meilisearch and Elasticsearch: 1. Filterable attributes must be defined explicitly. 2. No support for datetime objects, which must be converted to timestamps (with an extra field to store the timezone). 3. No special characters allowed in the primary key values, such that we must hash course IDs before we can use them as primary key values. Note that this PR does not introduce any breaking change. This is an opt-in engine that anyone is free to use. There is some setup work for every search feature: see the engine module documentation for more information. See the corresponding conversation here: openedx/frontend-app-authoring#1334 (comment) * fix: `make test` command Unit test command was failing because manage.py was pointing to an incorrect module by default. * feat: add convenient "make compile-requirements" command This command makes it possible to compile requirements without upgrading them. Note that the `make upgrade` command still works with this change. --------- Co-authored-by: Régis Behmo <[email protected]>
openedx · Oct 23, 2024 · 91686e9 · 91686e9
1 parent 8acaa21
commit 91686e9
Show file tree

Hide file tree

Showing 12 changed files with 1,079 additions and 145 deletions.
diff --git a/Makefile b/Makefile
@@ -34,23 +34,26 @@ test_with_es: clean test.start_elasticsearch
 	coverage run --source='.' manage.py test
 	make test.stop_elasticsearch
 
-upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade
-upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
+compile-requirements: export CUSTOM_COMPILE_COMMAND=make upgrade
+compile-requirements: ## Re-compile *.in requirements to *.txt (without upgrading)
 	pip install -qr requirements/pip-tools.txt
 	# Make sure to compile files after any other files they include!
 	pip-compile --rebuild --allow-unsafe --rebuild -o requirements/pip.txt requirements/pip.in
-	pip-compile --rebuild --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in
+	pip-compile --rebuild ${COMPILE_OPTS} -o requirements/pip-tools.txt requirements/pip-tools.in
 	pip install -qr requirements/pip.txt
 	pip install -qr requirements/pip-tools.txt
-	pip-compile --rebuild --upgrade -o requirements/base.txt requirements/base.in
-	pip-compile --rebuild --upgrade -o requirements/testing.txt requirements/testing.in
-	pip-compile --rebuild --upgrade -o requirements/quality.txt requirements/quality.in
-	pip-compile --rebuild --upgrade -o requirements/ci.txt requirements/ci.in
-	pip-compile --rebuild --upgrade -o requirements/dev.txt requirements/dev.in
+	pip-compile --rebuild ${COMPILE_OPTS} -o requirements/base.txt requirements/base.in
+	pip-compile --rebuild ${COMPILE_OPTS} -o requirements/testing.txt requirements/testing.in
+	pip-compile --rebuild ${COMPILE_OPTS} -o requirements/quality.txt requirements/quality.in
+	pip-compile --rebuild ${COMPILE_OPTS} -o requirements/ci.txt requirements/ci.in
+	pip-compile --rebuild ${COMPILE_OPTS} -o requirements/dev.txt requirements/dev.in
 	# Let tox control the Django version for tests
 	sed '/^[dD]jango==/d' requirements/testing.txt > requirements/testing.tmp
 	mv requirements/testing.tmp requirements/testing.txt
 
+upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
+	$(MAKE) compile-requirements COMPILE_OPTS="--upgrade"
+
 test: test_with_es ## run tests and generate coverage report
 
 install-local: ## installs your local edx-search into the LMS and CMS python virtualenvs

diff --git a/edxsearch/__init__.py b/edxsearch/__init__.py
@@ -1,3 +1,3 @@
 """ Container module for testing / demoing search """
 
-__version__ = '4.0.0'
+__version__ = '4.1.0'
diff --git a/edxsearch/settings.py b/edxsearch/settings.py
@@ -53,6 +53,7 @@
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
+    'eventtracking.django',
     'waffle',
 )
 
@@ -99,3 +100,31 @@
 # https://docs.djangoproject.com/en/1.6/howto/static-files/
 
 STATIC_URL = '/static/'
+
+# EVENT TRACKING #################################
+
+TRACK_MAX_EVENT = 50000
+
+TRACKING_BACKENDS = {
+    'logger': {
+        'ENGINE': 'track.backends.logger.LoggerBackend',
+        'OPTIONS': {
+            'name': 'tracking'
+        }
+    }
+}
+
+# We're already logging events, and we don't want to capture user
+# names/passwords.  Heartbeat events are likely not interesting.
+TRACKING_IGNORE_URL_PATTERNS = [r'^/event', r'^/login', r'^/heartbeat']
+
+EVENT_TRACKING_ENABLED = True
+EVENT_TRACKING_BACKENDS = {
+    'logger': {
+        'ENGINE': 'eventtracking.backends.logger.LoggerBackend',
+        'OPTIONS': {
+            'name': 'tracking',
+            'max_event_size': TRACK_MAX_EVENT,
+        }
+    }
+}
diff --git a/requirements/base.in b/requirements/base.in
@@ -15,3 +15,4 @@ Django                       # Web application framework
 elasticsearch>=7.8.0,<8.0.0
 edx-toggles
 event-tracking
+meilisearch
diff --git a/requirements/base.txt b/requirements/base.txt
@@ -6,18 +6,26 @@
 #
 amqp==5.2.0
     # via kombu
+annotated-types==0.7.0
+    # via pydantic
 asgiref==3.8.1
     # via django
 attrs==23.2.0
     # via openedx-events
 billiard==4.2.0
     # via celery
+camel-converter[pydantic]==4.0.1
+    # via meilisearch
 celery==5.4.0
     # via event-tracking
 certifi==2024.7.4
-    # via elasticsearch
+    # via
+    #   elasticsearch
+    #   requests
 cffi==1.16.0
     # via pynacl
+charset-normalizer==3.4.0
+    # via requests
 click==8.1.7
     # via
     #   celery
@@ -77,12 +85,16 @@ event-tracking==2.4.0
     #   -r requirements/base.in
 fastavro==1.9.5
     # via openedx-events
+idna==3.10
+    # via requests
 jinja2==3.1.4
     # via code-annotations
 kombu==5.3.7
     # via celery
 markupsafe==2.1.5
     # via jinja2
+meilisearch==0.31.5
+    # via -r requirements/base.in
 newrelic==9.12.0
     # via edx-django-utils
 openedx-events==9.11.0
@@ -95,6 +107,10 @@ psutil==6.0.0
     # via edx-django-utils
 pycparser==2.22
     # via cffi
+pydantic==2.9.2
+    # via camel-converter
+pydantic-core==2.23.4
+    # via pydantic
 pymongo==3.13.0
     # via
     #   edx-opaque-keys
@@ -109,6 +125,8 @@ pytz==2024.1
     # via event-tracking
 pyyaml==6.0.1
     # via code-annotations
+requests==2.32.3
+    # via meilisearch
 six==1.16.0
     # via
     #   edx-ccx-keys
@@ -124,11 +142,16 @@ stevedore==5.2.0
 text-unidecode==1.3
     # via python-slugify
 typing-extensions==4.12.2
-    # via edx-opaque-keys
+    # via
+    #   edx-opaque-keys
+    #   pydantic
+    #   pydantic-core
 tzdata==2024.1
     # via celery
 urllib3==1.26.19
-    # via elasticsearch
+    # via
+    #   elasticsearch
+    #   requests
 vine==5.1.0
     # via
     #   amqp

diff --git a/requirements/dev.txt b/requirements/dev.txt
@@ -9,6 +9,11 @@ amqp==5.2.0
     #   -r requirements/quality.txt
     #   -r requirements/testing.txt
     #   kombu
+annotated-types==0.7.0
+    # via
+    #   -r requirements/quality.txt
+    #   -r requirements/testing.txt
+    #   pydantic
 asgiref==3.8.1
     # via
     #   -r requirements/quality.txt
@@ -37,6 +42,11 @@ cachetools==5.4.0
     # via
     #   -r requirements/ci.txt
     #   tox
+camel-converter[pydantic]==4.0.1
+    # via
+    #   -r requirements/quality.txt
+    #   -r requirements/testing.txt
+    #   meilisearch
 celery==5.4.0
     # via
     #   -r requirements/quality.txt
@@ -47,6 +57,7 @@ certifi==2024.7.4
     #   -r requirements/quality.txt
     #   -r requirements/testing.txt
     #   elasticsearch
+    #   requests
 cffi==1.16.0
     # via
     #   -r requirements/quality.txt
@@ -56,6 +67,11 @@ chardet==5.2.0
     # via
     #   -r requirements/ci.txt
     #   tox
+charset-normalizer==3.4.0
+    # via
+    #   -r requirements/quality.txt
+    #   -r requirements/testing.txt
+    #   requests
 click==8.1.7
     # via
     #   -r requirements/pip-tools.txt
@@ -185,6 +201,11 @@ filelock==3.15.4
     #   -r requirements/ci.txt
     #   tox
     #   virtualenv
+idna==3.10
+    # via
+    #   -r requirements/quality.txt
+    #   -r requirements/testing.txt
+    #   requests
 iniconfig==2.0.0
     # via
     #   -r requirements/quality.txt
@@ -213,6 +234,10 @@ mccabe==0.7.0
     # via
     #   -r requirements/quality.txt
     #   pylint
+meilisearch==0.31.5
+    # via
+    #   -r requirements/quality.txt
+    #   -r requirements/testing.txt
 mock==5.1.0
     # via
     #   -r requirements/quality.txt
@@ -275,6 +300,16 @@ pycparser==2.22
     #   -r requirements/quality.txt
     #   -r requirements/testing.txt
     #   cffi
+pydantic==2.9.2
+    # via
+    #   -r requirements/quality.txt
+    #   -r requirements/testing.txt
+    #   camel-converter
+pydantic-core==2.23.4
+    # via
+    #   -r requirements/quality.txt
+    #   -r requirements/testing.txt
+    #   pydantic
 pylint==3.2.6
     # via
     #   -r requirements/quality.txt
@@ -344,6 +379,11 @@ pyyaml==6.0.1
     #   -r requirements/quality.txt
     #   -r requirements/testing.txt
     #   code-annotations
+requests==2.32.3
+    # via
+    #   -r requirements/quality.txt
+    #   -r requirements/testing.txt
+    #   meilisearch
 six==1.16.0
     # via
     #   -r requirements/quality.txt
@@ -380,6 +420,8 @@ typing-extensions==4.12.2
     #   -r requirements/quality.txt
     #   -r requirements/testing.txt
     #   edx-opaque-keys
+    #   pydantic
+    #   pydantic-core
 tzdata==2024.1
     # via
     #   -r requirements/quality.txt
@@ -390,6 +432,7 @@ urllib3==1.26.19
     #   -r requirements/quality.txt
     #   -r requirements/testing.txt
     #   elasticsearch
+    #   requests
 vine==5.1.0
     # via
     #   -r requirements/quality.txt

diff --git a/requirements/quality.txt b/requirements/quality.txt
@@ -8,6 +8,10 @@ amqp==5.2.0
     # via
     #   -r requirements/testing.txt
     #   kombu
+annotated-types==0.7.0
+    # via
+    #   -r requirements/testing.txt
+    #   pydantic
 asgiref==3.8.1
     # via
     #   -r requirements/testing.txt
@@ -24,6 +28,10 @@ billiard==4.2.0
     # via
     #   -r requirements/testing.txt
     #   celery
+camel-converter[pydantic]==4.0.1
+    # via
+    #   -r requirements/testing.txt
+    #   meilisearch
 celery==5.4.0
     # via
     #   -r requirements/testing.txt
@@ -32,10 +40,15 @@ certifi==2024.7.4
     # via
     #   -r requirements/testing.txt
     #   elasticsearch
+    #   requests
 cffi==1.16.0
     # via
     #   -r requirements/testing.txt
     #   pynacl
+charset-normalizer==3.4.0
+    # via
+    #   -r requirements/testing.txt
+    #   requests
 click==8.1.7
     # via
     #   -r requirements/testing.txt
@@ -130,6 +143,10 @@ fastavro==1.9.5
     # via
     #   -r requirements/testing.txt
     #   openedx-events
+idna==3.10
+    # via
+    #   -r requirements/testing.txt
+    #   requests
 iniconfig==2.0.0
     # via
     #   -r requirements/testing.txt
@@ -150,6 +167,8 @@ markupsafe==2.1.5
     #   jinja2
 mccabe==0.7.0
     # via pylint
+meilisearch==0.31.5
+    # via -r requirements/testing.txt
 mock==5.1.0
     # via -r requirements/testing.txt
 newrelic==9.12.0
@@ -188,6 +207,14 @@ pycparser==2.22
     # via
     #   -r requirements/testing.txt
     #   cffi
+pydantic==2.9.2
+    # via
+    #   -r requirements/testing.txt
+    #   camel-converter
+pydantic-core==2.23.4
+    # via
+    #   -r requirements/testing.txt
+    #   pydantic
 pylint==3.2.6
     # via
     #   edx-lint
@@ -233,6 +260,10 @@ pyyaml==6.0.1
     # via
     #   -r requirements/testing.txt
     #   code-annotations
+requests==2.32.3
+    # via
+    #   -r requirements/testing.txt
+    #   meilisearch
 six==1.16.0
     # via
     #   -r requirements/testing.txt
@@ -260,6 +291,8 @@ typing-extensions==4.12.2
     # via
     #   -r requirements/testing.txt
     #   edx-opaque-keys
+    #   pydantic
+    #   pydantic-core
 tzdata==2024.1
     # via
     #   -r requirements/testing.txt
@@ -268,6 +301,7 @@ urllib3==1.26.19
     # via
     #   -r requirements/testing.txt
     #   elasticsearch
+    #   requests
 vine==5.1.0
     # via
     #   -r requirements/testing.txt