From 44438143449e5d76e9d73a3458a139a115f9e4b7 Mon Sep 17 00:00:00 2001 From: Youcef Mammar Date: Mon, 25 Jan 2021 18:08:25 +0100 Subject: [PATCH 1/4] chore(deps): update algoliasearch to latest major --- algoliasearch_django/__init__.py | 3 +- .../management/commands/algolia_clearindex.py | 4 +- algoliasearch_django/models.py | 41 ++++++++++++------- algoliasearch_django/registration.py | 28 ++++++++----- algoliasearch_django/version.py | 2 +- setup.py | 2 +- tests/test_commands.py | 7 +++- tests/test_index.py | 25 ++++++----- tests/test_signal.py | 2 +- 9 files changed, 67 insertions(+), 47 deletions(-) diff --git a/algoliasearch_django/__init__.py b/algoliasearch_django/__init__.py index 67d28da..95c7c7d 100644 --- a/algoliasearch_django/__init__.py +++ b/algoliasearch_django/__init__.py @@ -30,7 +30,8 @@ delete_record = algolia_engine.delete_record update_records = algolia_engine.update_records raw_search = algolia_engine.raw_search -clear_index = algolia_engine.clear_index +clear_index = algolia_engine.clear_index # TODO: deprecate +clear_objects = algolia_engine.clear_objects reindex_all = algolia_engine.reindex_all # Default log handler diff --git a/algoliasearch_django/management/commands/algolia_clearindex.py b/algoliasearch_django/management/commands/algolia_clearindex.py index a3aee5e..a2c4ade 100644 --- a/algoliasearch_django/management/commands/algolia_clearindex.py +++ b/algoliasearch_django/management/commands/algolia_clearindex.py @@ -1,7 +1,7 @@ from django.core.management.base import BaseCommand from algoliasearch_django import get_registered_model -from algoliasearch_django import clear_index +from algoliasearch_django import clear_objects class Command(BaseCommand): @@ -18,5 +18,5 @@ def handle(self, *args, **options): options['model']): continue - clear_index(model) + clear_objects(model) self.stdout.write('\t* {}'.format(model.__name__)) diff --git a/algoliasearch_django/models.py b/algoliasearch_django/models.py index 82cdc46..66dcd2a 100644 --- a/algoliasearch_django/models.py +++ b/algoliasearch_django/models.py @@ -6,7 +6,7 @@ import logging import sys -from algoliasearch.helpers import AlgoliaException +from algoliasearch.exceptions import AlgoliaException from django.db.models.query_utils import DeferredAttribute from .settings import DEBUG @@ -179,8 +179,10 @@ def __init_index(self, client, model, settings): index_suffix=settings['INDEX_SUFFIX'] ) + self.tmp_index_name = tmp_index_name + self.__index = client.init_index(self.index_name) - self.__tmp_index = client.init_index(tmp_index_name) + self.__tmp_index = client.init_index(self.tmp_index_name) @staticmethod def _validate_geolocation(geolocation): @@ -400,10 +402,10 @@ def set_settings(self): logger.warning('SETTINGS NOT APPLIED ON %s: %s', self.model, e) - def clear_index(self): - """Clears the index.""" + def clear_objects(self): + """Clears all objects of an index.""" try: - self.__index.clear_index() + self.__index.clear_objects() logger.info('CLEAR INDEX %s', self.index_name) except AlgoliaException as e: if DEBUG: @@ -411,6 +413,10 @@ def clear_index(self): else: logger.warning('%s NOT CLEARED: %s', self.model, e) + def clear_index(self): + # TODO: add deprecated warning + self.clear_objects() + def wait_task(self, task_id): try: self.__index.wait_task(task_id) @@ -421,6 +427,11 @@ def wait_task(self, task_id): else: logger.warning('%s NOT WAIT: %s', self.model, e) + def delete(self): + self.__index.delete() + if self.__tmp_index: + self.__tmp_index.delete() + def reindex_all(self, batch_size=1000): """ Reindex all the records. @@ -457,13 +468,13 @@ def reindex_all(self, batch_size=1000): self.settings['slaves'] = [] logger.debug("REMOVE SLAVES FROM SETTINGS") - self.__tmp_index.wait_task(self.__tmp_index.set_settings(self.settings)['taskID']) + self.__tmp_index.set_settings(self.settings).wait() logger.debug('APPLY SETTINGS ON %s_tmp', self.index_name) rules = [] synonyms = [] - for r in self.__index.iter_rules(): + for r in self.__index.browse_rules(): rules.append(r) - for s in self.__index.iter_synonyms(): + for s in self.__index.browse_synonyms(): synonyms.append(s) if len(rules): logger.debug('Got rules for index %s: %s', self.index_name, rules) @@ -472,7 +483,7 @@ def reindex_all(self, batch_size=1000): logger.debug('Got synonyms for index %s: %s', self.index_name, rules) should_keep_synonyms = True - self.__tmp_index.clear_index() + self.__tmp_index.clear_objects() logger.debug('CLEAR INDEX %s_tmp', self.index_name) counts = 0 @@ -499,8 +510,8 @@ def reindex_all(self, batch_size=1000): logger.info('SAVE %d OBJECTS TO %s_tmp', len(batch), self.index_name) - self.__client.move_index(self.__tmp_index.index_name, - self.__index.index_name) + self.__client.move_index(self.tmp_index_name, + self.index_name) logger.info('MOVE INDEX %s_tmp TO %s', self.index_name, self.index_name) @@ -514,12 +525,12 @@ def reindex_all(self, batch_size=1000): if should_keep_replicas or should_keep_slaves: self.__index.set_settings(self.settings) if should_keep_rules: - response = self.__index.batch_rules(rules, forward_to_replicas=True) - self.__index.wait_task(response['taskID']) + response = self.__index.save_rules(rules, {'forwardToReplicas': True}) + response.wait() logger.info("Saved rules for index %s with response: {}".format(response), self.index_name) if should_keep_synonyms: - response = self.__index.batch_synonyms(synonyms, forward_to_replicas=True) - self.__index.wait_task(response['taskID']) + response = self.__index.save_synonyms(synonyms, {'forwardToReplicas': True}) + response.wait() logger.info("Saved synonyms for index %s with response: {}".format(response), self.index_name) return counts except AlgoliaException as e: diff --git a/algoliasearch_django/registration.py b/algoliasearch_django/registration.py index f36ae59..19e36e8 100644 --- a/algoliasearch_django/registration.py +++ b/algoliasearch_django/registration.py @@ -3,7 +3,8 @@ from django.db.models.signals import post_save from django.db.models.signals import pre_delete -from algoliasearch import algoliasearch +from algoliasearch.search_client import SearchClient +from algoliasearch.user_agent import UserAgent from .models import AlgoliaIndex from .settings import SETTINGS @@ -38,13 +39,14 @@ def __init__(self, settings=SETTINGS): self.__settings = settings self.__registered_models = {} - self.client = algoliasearch.Client(app_id, api_key) - self.client.set_extra_headers( - **{ - "User-Agent": "Algolia for Python (%s); Python (%s); Algolia for Django (%s); Django (%s)" - % (CLIENT_VERSION, python_version(), VERSION, django_version) - } - ) + self.client = SearchClient.create(app_id, api_key) + # UserAgent.add() + # self.client.set_extra_headers( + # **{ + # "User-Agent": "Algolia for Python (%s); Python (%s); Algolia for Django (%s); Django (%s)" + # % (CLIENT_VERSION, python_version(), VERSION, django_version) + # } + # ) def is_registered(self, model): """Checks whether the given models is registered with Algolia engine""" @@ -70,7 +72,7 @@ def register(self, model, index_cls=AlgoliaIndex, auto_indexing=None): self.__registered_models[model] = index_obj if (isinstance(auto_indexing, bool) and - auto_indexing) or self.__auto_indexing: + auto_indexing) or self.__auto_indexing: # Connect to the signalling framework. post_save.connect(self.__post_save_receiver, model) pre_delete.connect(self.__pre_delete_receiver, model) @@ -157,10 +159,14 @@ def raw_search(self, model, query='', params=None): adapter = self.get_adapter(model) return adapter.raw_search(query, params) - def clear_index(self, model): + def clear_objects(self, model): """Clears the index.""" adapter = self.get_adapter(model) - adapter.clear_index() + adapter.clear_objects() + + def clear_index(self, model): + # TODO: add deprecatd warning + self.clear_objects(model) def reindex_all(self, model, batch_size=1000): """ diff --git a/algoliasearch_django/version.py b/algoliasearch_django/version.py index 7250282..e1ab2c8 100644 --- a/algoliasearch_django/version.py +++ b/algoliasearch_django/version.py @@ -1 +1 @@ -VERSION = '1.7.3' +VERSION = '1.7.4' diff --git a/setup.py b/setup.py index 64cea44..ab39bef 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ version=VERSION, license='MIT License', packages=find_packages(exclude=['tests']), - install_requires=['django>=1.7', 'algoliasearch>=1.0,<2.0'], + install_requires=['django>=1.7', 'algoliasearch>=2.0,<3.0'], description='Algolia Search integration for Django', long_description=README, long_description_content_type='text/markdown', diff --git a/tests/test_commands.py b/tests/test_commands.py index 12db8fc..5b7e300 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -13,8 +13,11 @@ class CommandsTestCase(TestCase): @classmethod def tearDownClass(cls): - algolia_engine.client.delete_index(get_adapter(User).index_name) - algolia_engine.client.delete_index(get_adapter(Website).index_name) + user_index_name = get_adapter(User).index_name + website_index_name = get_adapter(Website).index_name + + algolia_engine.client.init_index(user_index_name).delete() + algolia_engine.client.init_index(website_index_name).delete() def setUp(self): # Create some records diff --git a/tests/test_index.py b/tests/test_index.py index f770d2e..ef17630 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -42,8 +42,8 @@ def setUp(self): ] def tearDown(self): - if hasattr(self, 'index') and hasattr(self.index, 'index_name'): - self.client.delete_index(self.index.index_name) + if hasattr(self, 'index'): + self.index.delete() def test_default_index_name(self): self.index = AlgoliaIndex(Website, self.client, settings.ALGOLIA) @@ -94,7 +94,7 @@ def test_tmp_index_name(self): with self.settings(ALGOLIA=algolia_settings): self.index = AlgoliaIndex(Website, self.client, settings.ALGOLIA) self.assertEqual( - self.index._AlgoliaIndex__tmp_index.index_name, + self.index.tmp_index_name, 'Website_tmp' ) @@ -104,7 +104,7 @@ def test_tmp_index_name(self): with self.settings(ALGOLIA=algolia_settings): self.index = AlgoliaIndex(Website, self.client, settings.ALGOLIA) self.assertEqual( - self.index._AlgoliaIndex__tmp_index.index_name, + self.index.tmp_index_name, 'prefix_Website_tmp' ) @@ -115,7 +115,7 @@ def test_tmp_index_name(self): with self.settings(ALGOLIA=algolia_settings): self.index = AlgoliaIndex(Website, self.client, settings.ALGOLIA) self.assertEqual( - self.index._AlgoliaIndex__tmp_index.index_name, + self.index.tmp_index_name, 'Website_tmp_suffix' ) @@ -126,7 +126,7 @@ def test_tmp_index_name(self): with self.settings(ALGOLIA=algolia_settings): self.index = AlgoliaIndex(Website, self.client, settings.ALGOLIA) self.assertEqual( - self.index._AlgoliaIndex__tmp_index.index_name, + self.index.tmp_index_name, 'prefix_Website_tmp_suffix' ) @@ -253,8 +253,8 @@ class WebsiteIndex(AlgoliaIndex): } } } - res = underlying_index.save_rule(rule) - self.index.wait_task(res['taskID']) + + underlying_index.save_rule(rule).wait() # When reindexing with no settings on the instance self.index = WebsiteIndex(Website, self.client, settings.ALGOLIA) @@ -267,7 +267,7 @@ def remove_metadata(rule): del copy["_metadata"] return copy - rules = [r for r in underlying_index.iter_rules()] + rules = [r for r in underlying_index.browse_rules()] rules = list(map(remove_metadata, rules)) self.assertEqual(len(rules), 1, "There should only be one rule") self.assertIn(rule, rules, "The existing rule should be kept over reindex") @@ -282,8 +282,7 @@ class WebsiteIndex(AlgoliaIndex): # Given some existing synonyms on the index synonym = {'objectID': 'street', 'type': 'altCorrection1', 'word': 'Street', 'corrections': ['St']} - task = underlying_index.batch_synonyms([synonym]) - underlying_index.wait_task(task['taskID']) + underlying_index.save_synonyms([synonym]).wait() # When reindexing with no settings on the instance self.index = WebsiteIndex(Website, self.client, settings.ALGOLIA) @@ -291,7 +290,7 @@ class WebsiteIndex(AlgoliaIndex): time.sleep(10) # FIXME: Refactor reindex_all to return taskID # Expect the synonyms to be kept across reindex - synonyms = [s for s in underlying_index.iter_synonyms()] + synonyms = [s for s in underlying_index.browse_synonyms()] self.assertEqual(len(synonyms), 1, "There should only be one synonym") self.assertIn(synonym, synonyms, "The existing synonym should be kept over reindex") @@ -661,7 +660,7 @@ class CyrillicIndex(AlgoliaIndex): self.user.bio = "крупнейших" self.user.save() self.index = CyrillicIndex(User, self.client, settings.ALGOLIA) - self.index.wait_task(self.index.save_record(self.user)["taskID"]) + self.index.save_record(self.user).wait() result = self.index.raw_search("крупнейших") self.assertEqual(result['nbHits'], 1, "Search should return one result") self.assertEqual(result['hits'][0]['name'], 'Algolia', "The result should be self.user") diff --git a/tests/test_signal.py b/tests/test_signal.py index 44dd802..5206a69 100644 --- a/tests/test_signal.py +++ b/tests/test_signal.py @@ -17,7 +17,7 @@ class SignalTestCase(TestCase): @classmethod def tearDownClass(cls): - algolia_engine.client.delete_index(get_adapter(Website).index_name) + get_adapter(Website).delete() def tearDown(self): clear_index(Website) From 97bb10ad7f67f579d164eabbc0579b820550b098 Mon Sep 17 00:00:00 2001 From: Youcef Mammar Date: Tue, 9 Mar 2021 14:21:19 +0100 Subject: [PATCH 2/4] fix(UserAgent): migrate user agent using new API exposed by 2.x client --- algoliasearch_django/registration.py | 12 +++--------- tests/test_engine.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/algoliasearch_django/registration.py b/algoliasearch_django/registration.py index 19e36e8..2e97568 100644 --- a/algoliasearch_django/registration.py +++ b/algoliasearch_django/registration.py @@ -9,12 +9,13 @@ from .models import AlgoliaIndex from .settings import SETTINGS from .version import VERSION -from algoliasearch.version import VERSION as CLIENT_VERSION -from platform import python_version from django import get_version as django_version logger = logging.getLogger(__name__) +UserAgent.add("Algolia for Django", VERSION) +UserAgent.add("Django", django_version()) + class AlgoliaEngineError(Exception): """Something went wrong with Algolia Engine.""" @@ -40,13 +41,6 @@ def __init__(self, settings=SETTINGS): self.__registered_models = {} self.client = SearchClient.create(app_id, api_key) - # UserAgent.add() - # self.client.set_extra_headers( - # **{ - # "User-Agent": "Algolia for Python (%s); Python (%s); Algolia for Django (%s); Django (%s)" - # % (CLIENT_VERSION, python_version(), VERSION, django_version) - # } - # ) def is_registered(self, model): """Checks whether the given models is registered with Algolia engine""" diff --git a/tests/test_engine.py b/tests/test_engine.py index a0baa76..1d97a79 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -1,8 +1,12 @@ import six +import re from django.conf import settings from django.test import TestCase +from algoliasearch.user_agent import UserAgent +from django import get_version as django_version +from algoliasearch_django.version import VERSION from algoliasearch_django import algolia_engine from algoliasearch_django import AlgoliaIndex from algoliasearch_django import AlgoliaEngine @@ -29,6 +33,14 @@ def test_init_exception(self): with self.assertRaises(AlgoliaEngineError): AlgoliaEngine(settings=settings.ALGOLIA) + def test_user_agent(self): + user_agent = UserAgent.get() + + parts = re.split('\s*;\s*', user_agent) + + self.assertIn('Django (%s)' % django_version(), parts) + self.assertIn('Algolia for Django (%s)' % VERSION, parts) + def test_auto_discover_indexes(self): """Test that the `index` module was auto-discovered and the models registered""" From 68c8e7e3dc29aa2609f31f2b0f69cb1ec53fed50 Mon Sep 17 00:00:00 2001 From: Youcef Mammar Date: Tue, 9 Mar 2021 14:31:59 +0100 Subject: [PATCH 3/4] fix: remove unecessary reformatting --- algoliasearch_django/registration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/algoliasearch_django/registration.py b/algoliasearch_django/registration.py index 2e97568..2616511 100644 --- a/algoliasearch_django/registration.py +++ b/algoliasearch_django/registration.py @@ -66,7 +66,7 @@ def register(self, model, index_cls=AlgoliaIndex, auto_indexing=None): self.__registered_models[model] = index_obj if (isinstance(auto_indexing, bool) and - auto_indexing) or self.__auto_indexing: + auto_indexing) or self.__auto_indexing: # Connect to the signalling framework. post_save.connect(self.__post_save_receiver, model) pre_delete.connect(self.__pre_delete_receiver, model) From 4b7913cc32e16ba1715fd9fa15628dd0577b367a Mon Sep 17 00:00:00 2001 From: Youcef Mammar Date: Wed, 17 Mar 2021 23:26:12 +0100 Subject: [PATCH 4/4] Update algoliasearch_django/version.py --- algoliasearch_django/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/algoliasearch_django/version.py b/algoliasearch_django/version.py index e1ab2c8..204a96f 100644 --- a/algoliasearch_django/version.py +++ b/algoliasearch_django/version.py @@ -1 +1 @@ -VERSION = '1.7.4' +VERSION = '2.0.0'