chore: add ELASTIC_SEARCH_INDEX_PREFIX setting to prefix indices

This helps with multitenancy so that indexes can be split per
tenant. Each index can be prefixed by the string defined
which will allow multiple tenants per ES cluster running the same
code.

README.md

@@ -34,6 +34,15 @@ where
 
 3. `search` - the operation to find matching documents within the index. `doc_type` is supported as an optional keyword parameter to return results only with a certain doc_type
 
+## Configuring for multi-tenancy
+
+The modules exposes a setting `ELASTIC_SEARCH_INDEX_PREFIX` to enable so that the indices for multiple clients do not collide.
+
+```python
+SearchEngine(index_name="test")
+```
+
+When invoked, this line will create an index named `test` on Elastic Search. Setting `ELASTIC_SEARCH_INDEX_PREFIX="client1_"` will instead create the index `client1_test`.
 
 ## Index documents
 Index documents are passed to the search application as python dictionaries, along with a `doc_type` document type, which is also optionally supported as a way to return only certain document types from a search.

edxsearch/__init__.py

@@ -1,3 +1,3 @@
 """ Container module for testing / demoing search """
 
-__version__ = '3.4.0'
+__version__ = '3.5.0'

search/elastic.py

@@ -304,16 +304,16 @@ def mappings(self):
         we'll load them again from Elasticsearch
         """
         # Try loading the mapping from the cache.
-        mapping = ElasticSearchEngine.get_mappings(self.index_name)
+        mapping = ElasticSearchEngine.get_mappings(self._prefixed_index_name)
 
         # Fall back to Elasticsearch
         if not mapping:
             mapping = self._es.indices.get_mapping(
-                index=self.index_name
-            ).get(self.index_name, {}).get("mappings", {})
+                index=self._prefixed_index_name
+            ).get(self._prefixed_index_name, {}).get("mappings", {})
             # Cache the mapping, if one was retrieved
             if mapping:
-                ElasticSearchEngine.set_mappings(self.index_name, mapping)
+                ElasticSearchEngine.set_mappings(self._prefixed_index_name, mapping)
 
         return mapping
 
@@ -323,14 +323,27 @@ def _clear_mapping(self):
 
         Next time ES mappings is are requested.
         """
-        ElasticSearchEngine.set_mappings(self.index_name, {})
+        ElasticSearchEngine.set_mappings(self._prefixed_index_name, {})
 
     def __init__(self, index=None):
         super().__init__(index)
         es_config = getattr(settings, "ELASTIC_SEARCH_CONFIG", [{}])
         self._es = getattr(settings, "ELASTIC_SEARCH_IMPL", Elasticsearch)(es_config)
-        if not self._es.indices.exists(index=self.index_name):
-            self._es.indices.create(index=self.index_name)
+        params = None
+
+        if not self._es.indices.exists(index=self._prefixed_index_name):
+            self._es.indices.create(index=self._prefixed_index_name, params=params)
+
+    @property
+    def _prefixed_index_name(self):
+        """
+        Property that returns the defined index_name with the configured
+        prefix.
+
+        To be used anywhere the index_name is required.
+        """
+        prefix = getattr(settings, "ELASTIC_SEARCH_INDEX_PREFIX", "")
+        return prefix + self.index_name
 
     def _check_mappings(self, body):
         """
@@ -396,9 +409,10 @@ def field_property(field_name, field_value):
         }
 
         if new_properties:
+
             self._es.indices.put_mapping(
-                index=self.index_name,
-                body={"properties": new_properties}
+                index=self._prefixed_index_name,
+                body={"properties": new_properties},
             )
             self._clear_mapping()
 
@@ -417,7 +431,7 @@ def index(self, sources, **kwargs):
                 id_ = source.get("id")
                 log.debug("indexing object with id %s", id_)
                 action = {
-                    "_index": self.index_name,
+                    "_index": self._prefixed_index_name,
                     "_id": id_,
                     "_source": source
                 }
@@ -437,14 +451,13 @@ def remove(self, doc_ids, **kwargs):
         """
         Implements call to remove the documents from the index
         """
-
         try:
             actions = []
             for doc_id in doc_ids:
                 log.debug("Removing document with id %s", doc_id)
                 action = {
                     "_op_type": "delete",
-                    "_index": self.index_name,
+                    "_index": self._prefixed_index_name,
                     "_id": doc_id
                 }
                 actions.append(action)
@@ -568,7 +581,6 @@ def search(self,
         """
 
         log.debug("searching index with %s", query_string)
-
         elastic_queries = []
         elastic_filters = []
 
@@ -642,7 +654,7 @@ def search(self,
             body["aggs"] = _process_aggregation_terms(aggregation_terms)
 
         try:
-            es_response = self._es.search(index=self.index_name, body=body, **kwargs)
+            es_response = self._es.search(index=self._prefixed_index_name, body=body, **kwargs)
         except exceptions.ElasticsearchException as ex:
             log.exception("error while searching index - %r", ex)
             raise

search/tests/test_engines.py

@@ -7,18 +7,35 @@
 import json
 import os
 from datetime import datetime
-
 from unittest.mock import patch
+
 from django.test import TestCase
 from django.test.utils import override_settings
 from elasticsearch import exceptions
 from elasticsearch.helpers import BulkIndexError
-
-from search.api import perform_search, NoSearchEngineError
+from search.api import NoSearchEngineError, perform_search
 from search.elastic import RESERVED_CHARACTERS
-from search.tests.mock_search_engine import MockSearchEngine, json_date_to_datetime
+from search.tests.mock_search_engine import (MockSearchEngine,
+                                             json_date_to_datetime)
 from search.tests.tests import MockSearchTests
-from search.tests.utils import ErroringElasticImpl, SearcherMixin
+from search.tests.utils import (TEST_INDEX_NAME, ErroringElasticImpl,
+                                SearcherMixin)
+
+
+@override_settings(ELASTIC_SEARCH_INDEX_PREFIX='prefixed_')
+@override_settings(SEARCH_ENGINE="search.tests.utils.ForceRefreshElasticSearchEngine")
+class ElasticSearchPrefixTests(MockSearchTests):
+    """
+    Override that runs the same tests for ElasticSearchTests,
+    but with a prefixed index name.
+    """
+
+    @property
+    def index_name(self):
+        """
+        The search index name to be used for this test.
+        """
+        return f"prefixed_{TEST_INDEX_NAME}"
 
 
 @override_settings(SEARCH_ENGINE="search.tests.utils.ForceRefreshElasticSearchEngine")

search/tests/tests.py

@@ -27,6 +27,14 @@
 @override_settings(MOCK_SEARCH_BACKING_FILE=None)
 class MockSearchTests(TestCase, SearcherMixin):
     """ Test operation of search activities """
+
+    @property
+    def index_name(self):
+        """
+        The search index name to be used for this test.
+        """
+        return TEST_INDEX_NAME
+
     @property
     def _is_elastic(self):
         """ check search engine implementation, to manage cleanup differently """
@@ -39,11 +47,11 @@ def setUp(self):
         if self._is_elastic:
             _elasticsearch = Elasticsearch()
             # Make sure that we are fresh
-            _elasticsearch.indices.delete(index=TEST_INDEX_NAME, ignore=[400, 404])
+            _elasticsearch.indices.delete(index=self.index_name, ignore=[400, 404])
 
             config_body = {}
             # ignore unexpected-keyword-arg; ES python client documents that it can be used
-            _elasticsearch.indices.create(index=TEST_INDEX_NAME, ignore=400, body=config_body)
+            _elasticsearch.indices.create(index=self.index_name, ignore=400, body=config_body)
         else:
             MockSearchEngine.destroy()
         self._searcher = None
@@ -55,7 +63,7 @@ def tearDown(self):
         if self._is_elastic:
             _elasticsearch = Elasticsearch()
             # ignore unexpected-keyword-arg; ES python client documents that it can be used
-            _elasticsearch.indices.delete(index=TEST_INDEX_NAME, ignore=[400, 404])
+            _elasticsearch.indices.delete(index=self.index_name, ignore=[400, 404])
         else:
             MockSearchEngine.destroy()