From 827c0d00f2e37026d68e3960f18e5b695c6e4c6e Mon Sep 17 00:00:00 2001 From: Casey Litton Date: Mon, 2 Nov 2020 11:37:04 -0800 Subject: [PATCH] SNO-179-allow-devserver-set-es-config-dir --- src/snovault/dev_servers.py | 21 +++-- src/snovault/tests/elasticsearch_fixture.py | 97 +++++++++++++++++---- src/snovault/tests/serverfixtures.py | 2 +- 3 files changed, 97 insertions(+), 23 deletions(-) diff --git a/src/snovault/dev_servers.py b/src/snovault/dev_servers.py index aaa00fcaaf..9ff7f55abb 100644 --- a/src/snovault/dev_servers.py +++ b/src/snovault/dev_servers.py @@ -60,6 +60,13 @@ def main(): parser.add_argument('--init', action="store_true", help="Init database") parser.add_argument('--load', action="store_true", help="Load test set") parser.add_argument('--datadir', default='/tmp/snovault', help="path to datadir") + parser.add_argument( + '-e', + '--es-version', + default=5, + type=int, + help='Override defult elasticsearch_fixture es version' + ) args = parser.parse_args() appsettings = get_appsettings(args.config_uri, name='app') @@ -77,17 +84,19 @@ def main(): from snovault.elasticsearch import create_mapping datadir = os.path.abspath(args.datadir) pgdata = os.path.join(datadir, 'pgdata') - esdata = os.path.join(datadir, 'esdata') redisdata = os.path.join(datadir, 'redisdata') if args.clear: - for dirname in [pgdata, esdata, redisdata]: - if os.path.exists(dirname): - shutil.rmtree(dirname) + if os.path.exists(args.datadir): + shutil.rmtree(args.datadir) if args.init: postgresql_fixture.initdb(pgdata, echo=True) - postgres = postgresql_fixture.server_process(pgdata, echo=True) - elasticsearch = elasticsearch_fixture.server_process(esdata, echo=True) + elasticsearch = elasticsearch_fixture.server_process( + datadir, + clear=args.clear, + echo=True, + version=args.es_version, + ) nginx = nginx_server_process(echo=True) redis_config_path = redis_storage_fixture.initdb(redisdata, local_storage_port, echo=True) redis = redis_storage_fixture.server_process(redis_config_path, local_storage_port, local_storage_redis_index, echo=True) diff --git a/src/snovault/tests/elasticsearch_fixture.py b/src/snovault/tests/elasticsearch_fixture.py index 3994e099e6..58bb8a7cc0 100644 --- a/src/snovault/tests/elasticsearch_fixture.py +++ b/src/snovault/tests/elasticsearch_fixture.py @@ -1,4 +1,5 @@ import os.path +import shutil import sys from time import sleep try: @@ -7,34 +8,98 @@ import subprocess -def server_process(datadir, host='127.0.0.1', port=9201, prefix='', echo=False): +# This is required for local osx testing, circle version is sepearte +_OSX_ES_VERSION = 5 +# These are set in .circleci/config +_CIRCLE_BUILD = os.environ.get('BASH_ENV') == '/home/circleci/.bashrc' +_CIRCLE_ES_VERSION = os.environ.get('ES_MAJOR_VERSION', str(_OSX_ES_VERSION)) +# Encoded configuraion repo +_ENCD_CONFIG_DIR = os.environ.get('ENCD_CONFIG_DIR') + + +def _get_args_and_env(esdata, esdata_override, kwargs): + env = os.environ.copy() args = [ - os.path.join(prefix, 'elasticsearch'), - '-Enetwork.host=%s' % host, - '-Ehttp.port=%d' % port, - '-Epath.data=%s' % os.path.join(datadir, 'data'), - '-Epath.logs=%s' % os.path.join(datadir, 'logs'), + os.path.join(kwargs.get('prefix', ''), 'elasticsearch'), ] - if os.environ.get('TRAVIS'): - print('IN TRAVIS') - echo = True - args.append('-Epath.conf=%s/conf' % os.environ['TRAVIS_BUILD_DIR']) - elif os.path.exists('/etc/elasticsearch'): - print('NOT IN TRAVIS') + if esdata_override: + # ES configuration dir is specified, not default location. + # 'esdata' is ignored and kwargs values are overidden by elasticsearch.yml + esconfig = f"{esdata_override}/config" + if kwargs['version'] <= 5: + # How elasticsearch 5 sets config path + args.append(f"-Epath.conf={esconfig}") + else: + # How elasticsearch 6+ sets config path + env["ES_PATH_CONF"] = esconfig + jvm_options = _get_jvm_options(f"{esconfig}/jvm.options") + if jvm_options: + env['ES_JAVA_OPTS'] = jvm_options + return args, env + # Default 'esdata' location + args.extend([ + '-Enetwork.host=%s' % kwargs.get('host', '127.0.0.1'), + '-Ehttp.port=%d' % kwargs.get('port', 9201), + '-Epath.data=%s' % os.path.join(esdata, 'data'), + '-Epath.logs=%s' % os.path.join(esdata, 'logs'), + ]) + if _CIRCLE_BUILD and int(_CIRCLE_ES_VERSION) == 5: args.append('-Epath.conf=./conf') - print(args) + return args, env + + +def _get_config_override(esdata, kwargs): + if _ENCD_CONFIG_DIR: + # If found, elasticsearch.yml config overrides vars like logs/data/host/port/etc... + es_data_dir = f"{_ENCD_CONFIG_DIR}/elasticsearch/es{kwargs['version']}" + if kwargs.get('local_test', False): + es_data_dir += 'test' + if os.path.exists(f"{es_data_dir}/config/elasticsearch.yml"): + return es_data_dir + return None + + +def _get_jvm_options(jvm_options_path): + with open(jvm_options_path, 'r') as fileh: + return ' '.join([ + line.strip() + for line in fileh.readlines() + if line[0] == '-' + ]) + + +def _clear(esdata, esdata_override): + rm_dirs = [esdata] + if esdata_override: + esconfig = f"{esdata_override}/config" + rm_dirs.extend([f"{esdata_override}/data", f"{esdata_override}/logs", f"{esconfig}/scripts"]) + es_keystore_path = f"{esconfig}/elasticsearch.keystore" + if os.path.exists(es_keystore_path): + os.remove(es_keystore_path) + for dir_path in rm_dirs: + if os.path.exists(dir_path): + shutil.rmtree(dir_path) + + +def server_process(datadir, **kwargs): + kwargs['version'] = kwargs.get('version', _OSX_ES_VERSION) + esdata = os.path.join(datadir, 'data') + esdata_override = _get_config_override(esdata, kwargs) + if kwargs.get('clear', False): + _clear(esdata, esdata_override) + args, env = _get_args_and_env(esdata, esdata_override, kwargs) process = subprocess.Popen( args, close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + env=env, ) SUCCESS_LINE = b'started\n' - lines = [] for line in iter(process.stdout.readline, b''): - if echo: + if kwargs.get('echo', False): sys.stdout.write(line.decode('utf-8')) lines.append(line) if line.endswith(SUCCESS_LINE): @@ -45,7 +110,7 @@ def server_process(datadir, host='127.0.0.1', port=9201, prefix='', echo=False): msg = ('Process return code: %d\n' % code) + b''.join(lines).decode('utf-8') raise Exception(msg) - if not echo: + if not kwargs.get('echo', False): process.stdout.close() print('returning process') return process diff --git a/src/snovault/tests/serverfixtures.py b/src/snovault/tests/serverfixtures.py index 56935c76b8..8c2bd48227 100644 --- a/src/snovault/tests/serverfixtures.py +++ b/src/snovault/tests/serverfixtures.py @@ -74,7 +74,7 @@ def elasticsearch_server(request, elasticsearch_host_port): host, port = elasticsearch_host_port tmpdir = request.config._tmpdirhandler.mktemp('elasticsearch', numbered=True) tmpdir = str(tmpdir) - process = server_process(str(tmpdir), host=host, port=9201, echo=False) + process = server_process(str(tmpdir), local_test=True, host=host, port=9201, echo=False) print('PORT CHANGED') yield 'http://%s:%d' % (host, 9201)