From f03fb8ea2846e0626754d727edd8b72331926e27 Mon Sep 17 00:00:00 2001 From: Keith Grootboom Date: Wed, 9 Nov 2022 15:33:26 +0200 Subject: [PATCH] chore: add ELASTIC_SEARCH_INDEX_PREFIX setting to prefix indices This helps with multitenancy so that indexes can be split per tenant. Each index can be prefixed by the string defined which will allow multiple tenants per ES cluster running the same code. --- README.md | 9 ++++++++ edxsearch/__init__.py | 2 +- search/elastic.py | 40 +++++++++++++++++++++++------------- search/tests/test_engines.py | 27 +++++++++++++++++++----- search/tests/tests.py | 14 ++++++++++--- 5 files changed, 69 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 007e7a7b..f3abad1c 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,15 @@ where 3. `search` - the operation to find matching documents within the index. `doc_type` is supported as an optional keyword parameter to return results only with a certain doc_type +## Configuring for multi-tenancy + +The modules exposes a setting `ELASTIC_SEARCH_INDEX_PREFIX` to enable so that the indices for multiple clients do not collide. + +```python +SearchEngine(index_name="test") +``` + +When invoked, this line will create an index named `test` on Elastic Search. Setting `ELASTIC_SEARCH_INDEX_PREFIX="client1_"` will instead create the index `client1_test`. ## Index documents Index documents are passed to the search application as python dictionaries, along with a `doc_type` document type, which is also optionally supported as a way to return only certain document types from a search. diff --git a/edxsearch/__init__.py b/edxsearch/__init__.py index 2432599e..a33f2731 100644 --- a/edxsearch/__init__.py +++ b/edxsearch/__init__.py @@ -1,3 +1,3 @@ """ Container module for testing / demoing search """ -__version__ = '3.4.0' +__version__ = '3.5.0' diff --git a/search/elastic.py b/search/elastic.py index 701378cd..f895564a 100644 --- a/search/elastic.py +++ b/search/elastic.py @@ -304,16 +304,16 @@ def mappings(self): we'll load them again from Elasticsearch """ # Try loading the mapping from the cache. - mapping = ElasticSearchEngine.get_mappings(self.index_name) + mapping = ElasticSearchEngine.get_mappings(self._prefixed_index_name) # Fall back to Elasticsearch if not mapping: mapping = self._es.indices.get_mapping( - index=self.index_name - ).get(self.index_name, {}).get("mappings", {}) + index=self._prefixed_index_name + ).get(self._prefixed_index_name, {}).get("mappings", {}) # Cache the mapping, if one was retrieved if mapping: - ElasticSearchEngine.set_mappings(self.index_name, mapping) + ElasticSearchEngine.set_mappings(self._prefixed_index_name, mapping) return mapping @@ -323,14 +323,27 @@ def _clear_mapping(self): Next time ES mappings is are requested. """ - ElasticSearchEngine.set_mappings(self.index_name, {}) + ElasticSearchEngine.set_mappings(self._prefixed_index_name, {}) def __init__(self, index=None): super().__init__(index) es_config = getattr(settings, "ELASTIC_SEARCH_CONFIG", [{}]) self._es = getattr(settings, "ELASTIC_SEARCH_IMPL", Elasticsearch)(es_config) - if not self._es.indices.exists(index=self.index_name): - self._es.indices.create(index=self.index_name) + params = None + + if not self._es.indices.exists(index=self._prefixed_index_name): + self._es.indices.create(index=self._prefixed_index_name, params=params) + + @property + def _prefixed_index_name(self): + """ + Property that returns the defined index_name with the configured + prefix. + + To be used anywhere the index_name is required. + """ + prefix = getattr(settings, "ELASTIC_SEARCH_INDEX_PREFIX", "") + return prefix + self.index_name def _check_mappings(self, body): """ @@ -396,9 +409,10 @@ def field_property(field_name, field_value): } if new_properties: + self._es.indices.put_mapping( - index=self.index_name, - body={"properties": new_properties} + index=self._prefixed_index_name, + body={"properties": new_properties}, ) self._clear_mapping() @@ -417,7 +431,7 @@ def index(self, sources, **kwargs): id_ = source.get("id") log.debug("indexing object with id %s", id_) action = { - "_index": self.index_name, + "_index": self._prefixed_index_name, "_id": id_, "_source": source } @@ -437,14 +451,13 @@ def remove(self, doc_ids, **kwargs): """ Implements call to remove the documents from the index """ - try: actions = [] for doc_id in doc_ids: log.debug("Removing document with id %s", doc_id) action = { "_op_type": "delete", - "_index": self.index_name, + "_index": self._prefixed_index_name, "_id": doc_id } actions.append(action) @@ -568,7 +581,6 @@ def search(self, """ log.debug("searching index with %s", query_string) - elastic_queries = [] elastic_filters = [] @@ -642,7 +654,7 @@ def search(self, body["aggs"] = _process_aggregation_terms(aggregation_terms) try: - es_response = self._es.search(index=self.index_name, body=body, **kwargs) + es_response = self._es.search(index=self._prefixed_index_name, body=body, **kwargs) except exceptions.ElasticsearchException as ex: log.exception("error while searching index - %r", ex) raise diff --git a/search/tests/test_engines.py b/search/tests/test_engines.py index 2b4f32bd..659d4cc9 100644 --- a/search/tests/test_engines.py +++ b/search/tests/test_engines.py @@ -7,18 +7,35 @@ import json import os from datetime import datetime - from unittest.mock import patch + from django.test import TestCase from django.test.utils import override_settings from elasticsearch import exceptions from elasticsearch.helpers import BulkIndexError - -from search.api import perform_search, NoSearchEngineError +from search.api import NoSearchEngineError, perform_search from search.elastic import RESERVED_CHARACTERS -from search.tests.mock_search_engine import MockSearchEngine, json_date_to_datetime +from search.tests.mock_search_engine import (MockSearchEngine, + json_date_to_datetime) from search.tests.tests import MockSearchTests -from search.tests.utils import ErroringElasticImpl, SearcherMixin +from search.tests.utils import (TEST_INDEX_NAME, ErroringElasticImpl, + SearcherMixin) + + +@override_settings(ELASTIC_SEARCH_INDEX_PREFIX='prefixed_') +@override_settings(SEARCH_ENGINE="search.tests.utils.ForceRefreshElasticSearchEngine") +class ElasticSearchPrefixTests(MockSearchTests): + """ + Override that runs the same tests for ElasticSearchTests, + but with a prefixed index name. + """ + + @property + def index_name(self): + """ + The search index name to be used for this test. + """ + return f"prefixed_{TEST_INDEX_NAME}" @override_settings(SEARCH_ENGINE="search.tests.utils.ForceRefreshElasticSearchEngine") diff --git a/search/tests/tests.py b/search/tests/tests.py index bf7a7259..62ea00cd 100644 --- a/search/tests/tests.py +++ b/search/tests/tests.py @@ -27,6 +27,14 @@ @override_settings(MOCK_SEARCH_BACKING_FILE=None) class MockSearchTests(TestCase, SearcherMixin): """ Test operation of search activities """ + + @property + def index_name(self): + """ + The search index name to be used for this test. + """ + return TEST_INDEX_NAME + @property def _is_elastic(self): """ check search engine implementation, to manage cleanup differently """ @@ -39,11 +47,11 @@ def setUp(self): if self._is_elastic: _elasticsearch = Elasticsearch() # Make sure that we are fresh - _elasticsearch.indices.delete(index=TEST_INDEX_NAME, ignore=[400, 404]) + _elasticsearch.indices.delete(index=self.index_name, ignore=[400, 404]) config_body = {} # ignore unexpected-keyword-arg; ES python client documents that it can be used - _elasticsearch.indices.create(index=TEST_INDEX_NAME, ignore=400, body=config_body) + _elasticsearch.indices.create(index=self.index_name, ignore=400, body=config_body) else: MockSearchEngine.destroy() self._searcher = None @@ -55,7 +63,7 @@ def tearDown(self): if self._is_elastic: _elasticsearch = Elasticsearch() # ignore unexpected-keyword-arg; ES python client documents that it can be used - _elasticsearch.indices.delete(index=TEST_INDEX_NAME, ignore=[400, 404]) + _elasticsearch.indices.delete(index=self.index_name, ignore=[400, 404]) else: MockSearchEngine.destroy()