From 2ed56bb388bf8d49f10dc49ea3dd012394b66657 Mon Sep 17 00:00:00 2001 From: drfho Date: Fri, 5 Jul 2024 00:19:16 +0200 Subject: [PATCH] opensearch: added performance test button to admin gui Ref: https://github.com/idasm-unibe-ch/unibe-cms/pull/671#event-13395102955 --- .../opensearch_connector/__init__.py | 9 ++ .../manage_opensearch_test.py | 102 ++++++++++++++++++ .../opensearch_connector/opensearch_query.py | 6 +- .../opensearch_suggest.py | 6 +- .../manage_zcatalog_connector.zpt | 23 +++- 5 files changed, 142 insertions(+), 4 deletions(-) create mode 100644 Products/zms/conf/metaobj_manager/com.zms.catalog.opensearch/opensearch_connector/manage_opensearch_test.py diff --git a/Products/zms/conf/metaobj_manager/com.zms.catalog.opensearch/opensearch_connector/__init__.py b/Products/zms/conf/metaobj_manager/com.zms.catalog.opensearch/opensearch_connector/__init__.py index 9a17ad4d0..048b4ae4f 100644 --- a/Products/zms/conf/metaobj_manager/com.zms.catalog.opensearch/opensearch_connector/__init__.py +++ b/Products/zms/conf/metaobj_manager/com.zms.catalog.opensearch/opensearch_connector/__init__.py @@ -112,6 +112,15 @@ class Attrs: ,"repetitive":0 ,"type":"External Method"} + manage_opensearch_test = {"default":"" + ,"id":"manage_opensearch_test" + ,"keys":[] + ,"mandatory":0 + ,"multilang":0 + ,"name":"Test: OpenSearch connection" + ,"repetitive":0 + ,"type":"External Method"} + opensearch_query = {"default":"" ,"id":"opensearch_query" ,"keys":[] diff --git a/Products/zms/conf/metaobj_manager/com.zms.catalog.opensearch/opensearch_connector/manage_opensearch_test.py b/Products/zms/conf/metaobj_manager/com.zms.catalog.opensearch/opensearch_connector/manage_opensearch_test.py new file mode 100644 index 000000000..abdcdbd83 --- /dev/null +++ b/Products/zms/conf/metaobj_manager/com.zms.catalog.opensearch/opensearch_connector/manage_opensearch_test.py @@ -0,0 +1,102 @@ +from opensearchpy import OpenSearch +from urllib.parse import urlparse +import json +import traceback +import time + +def manage_opensearch_test(self): + client = get_opensearch_client(self) + request = self.REQUEST + call_amount = int(request.get('calls', 10)) + request.set('q', request.get('q', 'Universität')) + prettify = bool(int(request.get('prettify',0))) + response = {} + try: + response['status'] = 200 + response['clusterInfo'] = client.info() + tests = {} + tests['options'] = {} + tests['options']['query'] = request.get('q') + tests['options']['callAmount'] = call_amount + response['tests'] = tests + # Call OpenSearch query multiple times + query_result = {} + query_result['performance'] = {} + query_result_took = query_result['performance']['took'] = {} + query_result_took['sum'] = 0 + query_start = time.time() + for i in range(call_amount): + stringified_query_response = self.opensearch_query() + query_response = json.loads(stringified_query_response) + query_result_took['sum'] += query_response['took'] + if i == 0: + query_result['response'] = query_response + query_end = time.time() + query_result_took['average'] = query_result_took['sum'] / call_amount + query_result_elapsed_time = query_result['performance']['ellapsedTime'] = {} + query_result_elapsed_time['sum'] = (query_end - query_start) * 1000 + query_result_elapsed_time['average'] = query_result_elapsed_time['sum'] / call_amount + tests['opensearchQuery'] = query_result + + # Call OpenSearch suggest multiple times + suggest_result = {} + suggest_result['performance'] = {} + suggest_start = time.time() + for i in range(call_amount): + stringified_suggest_response = self.opensearch_suggest() + suggest_response = json.loads(stringified_suggest_response) + if i == 0: + suggest_result['response'] = suggest_response + suggest_end = time.time() + suggest_result_elapsed_time = suggest_result['performance']['ellapsedTime'] = {} + suggest_result_elapsed_time['sum'] = (suggest_end - suggest_start) * 1000 + suggest_result_elapsed_time['average'] = suggest_result_elapsed_time['sum'] / call_amount + tests['opensearchSuggest'] = suggest_result + except Exception as e: + response['status'] = 500 + response['error'] = str(e) + response['trace'] = traceback.format_exc() + + if prettify: + resp_text = json.dumps(response, indent=4) + else: + resp_text = json.dumps(response) + + return resp_text + + +def get_opensearch_client(self): + # ${opensearch.url:https://localhost:9200, https://localhost:9201} + # ${opensearch.username:admin} + # ${opensearch.password:admin} + # ${opensearch.ssl.verify:} + url_string = self.getConfProperty('opensearch.url') + urls = [url.strip().rstrip('/') for url in url_string.split(',')] + hosts = [] + use_ssl = False + # Process (multiple) url(s) (host, port, ssl) + if not urls: + return None + else: + for url in urls: + hosts.append( { \ + 'host':urlparse(url).hostname, \ + 'port':urlparse(url).port } \ + ) + if urlparse(url).scheme=='https': + use_ssl = True + verify = bool(self.getConfProperty('opensearch.ssl.verify', False)) + username = self.getConfProperty('opensearch.username', 'admin') + password = self.getConfProperty('opensearch.password', 'admin') + auth = (username,password) + + client = OpenSearch( + hosts = hosts, + http_compress = False, # enables gzip compression for request bodies + http_auth = auth, + use_ssl = use_ssl, + verify_certs = verify, + ssl_assert_hostname = verify, + ssl_show_warn = False, + ) + return client diff --git a/Products/zms/conf/metaobj_manager/com.zms.catalog.opensearch/opensearch_connector/opensearch_query.py b/Products/zms/conf/metaobj_manager/com.zms.catalog.opensearch/opensearch_connector/opensearch_query.py index 21337af2c..58130e210 100644 --- a/Products/zms/conf/metaobj_manager/com.zms.catalog.opensearch/opensearch_connector/opensearch_query.py +++ b/Products/zms/conf/metaobj_manager/com.zms.catalog.opensearch/opensearch_connector/opensearch_query.py @@ -50,6 +50,7 @@ def opensearch_query( self, REQUEST=None): home_id = request.get('home_id', '') multisite_search = int(request.get('multisite_search', 1)) multisite_exclusions = request.get('multisite_exclusions', '').split(',') + prettify = bool(int(request.get('prettify',0))) index_names = [] # Search in a specific index given by Request-parameter facet if request.get('facet') not in ['all','undefined', None, '']: @@ -176,7 +177,10 @@ def opensearch_query( self, REQUEST=None): resp_text = '' try: response = client.search(body = json.dumps(query), index = index_names) - resp_text = json.dumps(response) + if prettify: + resp_text = json.dumps(response, indent=4) + else: + resp_text = json.dumps(response) except opensearchpy.exceptions.RequestError as e: resp_text = '//%s'%(e.error) diff --git a/Products/zms/conf/metaobj_manager/com.zms.catalog.opensearch/opensearch_connector/opensearch_suggest.py b/Products/zms/conf/metaobj_manager/com.zms.catalog.opensearch/opensearch_connector/opensearch_suggest.py index 007410cdc..472da3968 100644 --- a/Products/zms/conf/metaobj_manager/com.zms.catalog.opensearch/opensearch_connector/opensearch_suggest.py +++ b/Products/zms/conf/metaobj_manager/com.zms.catalog.opensearch/opensearch_connector/opensearch_suggest.py @@ -137,6 +137,7 @@ def opensearch_suggest(self, REQUEST=None): q = request.get('q','Lorem') qsize = request.get('qsize', 12) debug = bool(int(request.get('debug',0))) + prettify = bool(int(request.get('prettify',0))) # Get configured or default fieldsets that are used for extracting suggest terms fieldsets = get_suggest_fieldsets(self) @@ -150,5 +151,8 @@ def opensearch_suggest(self, REQUEST=None): # Finally sort all terms and return as JSON-textTestBan suggest_terms = sorted(set(suggest_terms), key=lambda x: x.lower()) - resp_text = json.dumps(suggest_terms, indent=2) + if prettify: + resp_text = json.dumps(suggest_terms, indent=4) + else: + resp_text = json.dumps(suggest_terms) return resp_text diff --git a/Products/zms/zpt/ZMSZCatalogAdapter/manage_zcatalog_connector.zpt b/Products/zms/zpt/ZMSZCatalogAdapter/manage_zcatalog_connector.zpt index dad6f1bcb..d92cb8ba4 100644 --- a/Products/zms/zpt/ZMSZCatalogAdapter/manage_zcatalog_connector.zpt +++ b/Products/zms/zpt/ZMSZCatalogAdapter/manage_zcatalog_connector.zpt @@ -210,6 +210,25 @@ +
+ +
+
+ + +
+ +
+
+
+
@@ -217,7 +236,7 @@
@@ -231,7 +250,7 @@