From 3c1e3e3856ddea571eedbbecfe52218430cada94 Mon Sep 17 00:00:00 2001 From: Ulugbek Baymuradov Date: Mon, 28 Sep 2020 20:07:58 +0500 Subject: [PATCH] configurable shards --- base.ini | 2 ++ src/snovault/elasticsearch/create_mapping.py | 12 +++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/base.ini b/base.ini index aad6e6f68b..996d808e91 100644 --- a/base.ini +++ b/base.ini @@ -5,6 +5,8 @@ retry.attempts = 3 file_upload_bucket = snowflakes-files-dev file_upload_profile_name = snowflakes-files-upload elasticsearch.server = localhost:9201 +elasticsearch.shards.primary = 1 +elasticsearch.shards.replicate = 0 aws_ip_ranges_path = %(here)s/aws-ip-ranges.json download_proxy = https://download.encodeproject.org/ annotations_path = %(here)s/annotations.json diff --git a/src/snovault/elasticsearch/create_mapping.py b/src/snovault/elasticsearch/create_mapping.py index cd844267f7..56915e0789 100644 --- a/src/snovault/elasticsearch/create_mapping.py +++ b/src/snovault/elasticsearch/create_mapping.py @@ -164,13 +164,13 @@ def schema_mapping(name, schema): } -def index_settings(): +def index_settings(primary_shards=3, replicate_shards=1): return { 'settings': { 'index.max_result_window': 99999, 'index.mapping.total_fields.limit': 5000, - 'index.number_of_shards': 1, - 'index.number_of_replicas': 0, + 'index.number_of_shards': primary_shards, + 'index.number_of_replicas': replicate_shards, 'index.max_ngram_diff': 32, 'analysis': { 'filter': { @@ -484,6 +484,8 @@ def create_snovault_index_alias(es, indices): def run(app, collections=None, dry_run=False): index = app.registry.settings['snovault.elasticsearch.index'] registry = app.registry + primary_shards = registry.settings.get('elasticsearch.shards.primary', 3) + replicate_shards = registry.settings.get('elasticsearch.shards.replicate', 1) if not dry_run: es = app.registry[ELASTIC_SEARCH] print('CREATE MAPPING RUNNING') @@ -503,9 +505,9 @@ def run(app, collections=None, dry_run=False): if mapping is None: continue # Testing collections if dry_run: - print(json.dumps(sorted_dict({index: {doc_type: mapping}}), indent=4)) + print(json.dumps(sorted_dict({index: {collection_name: mapping}}), indent=4)) continue - create_elasticsearch_index(es, index, index_settings()) + create_elasticsearch_index(es, index, index_settings(primary_shards, replicate_shards)) set_index_mapping(es, index, mapping) if collection_name != 'meta': indices.append(index)