From 3339bfaf2d5b8189c0e201af22e6d9e4df76c298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eddy=20=E2=88=86?= <12048448+ome9ax@users.noreply.github.com> Date: Tue, 17 Aug 2021 15:11:27 +0100 Subject: [PATCH] [coverage] bump version 0.0.6 changelog update: Much more specs and tests, coverage increased to `96.91%` (#15) * [coverage] bump version 0.0.6 changelog update: Much more specs and tests, coverage increased to `96.91%` * [coverage] bump version 0.0.6 changelog update * [coverage] Draft4Validator FormatChecker comeback * [coverage] test_init s3 import farewell * [coverage] let anything but InvalidOperation raised Exception slip by Draft4Validator FormatChecker * [coverage] lower coverage threashold to 90% * [coverage] s3 tests & spec, `96.91%` coverage back for good --- CHANGELOG.md | 25 ++-- README.md | 4 +- setup.cfg | 2 +- target_s3_jsonl/__init__.py | 42 +++--- target_s3_jsonl/s3.py | 8 +- tests/resources/config.json | 1 - ...fig_local_false.json => config_local.json} | 2 +- tests/resources/messages.json | 6 + tests/test_init.py | 132 ++++++++++++++---- tests/test_s3.py | 93 ++++++++++++ 10 files changed, 246 insertions(+), 69 deletions(-) rename tests/resources/{config_local_false.json => config_local.json} (93%) create mode 100644 tests/resources/messages.json create mode 100644 tests/test_s3.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 092d660..2c60b70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,16 @@ # Change Log -## [v0.0.5.2](https://github.com/ome9ax/target-s3-jsonl/tree/v0.0.5.2) (2021-08-13) -[Full Changelog](https://github.com/ome9ax/target-s3-jsonl/tree/v0.0.5.1...v0.0.5.2) +## [0.0.6](https://github.com/ome9ax/target-s3-jsonl/tree/0.0.6) (2021-08-17) +[Full Changelog](https://github.com/ome9ax/target-s3-jsonl/tree/0.0.5.2...0.0.6) + +### Closed issues: +- Much more specs and tests, coverage increased to `96.91%` + +### Merged pull requests: +- [[coverage] bump version 0.0.6 changelog update: Much more specs and tests, coverage increased to `96.91%`](https://github.com/ome9ax/target-s3-jsonl/pull/15) + +## [0.0.5.2](https://github.com/ome9ax/target-s3-jsonl/tree/0.0.5.2) (2021-08-13) +[Full Changelog](https://github.com/ome9ax/target-s3-jsonl/tree/0.0.5.1...0.0.5.2) ### New features: - replace `io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8')` with `sys.stdin` as it's already natively defined as `<_io.TextIOWrapper name='' mode='r' encoding='utf-8'>` @@ -9,8 +18,8 @@ ### Merged pull requests: - [[readlines] replace `io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8')` with `sys.stdin`](https://github.com/ome9ax/target-s3-jsonl/pull/13) -## [v0.0.5.1](https://github.com/ome9ax/target-s3-jsonl/tree/v0.0.5.1) (2021-08-12) -[Full Changelog](https://github.com/ome9ax/target-s3-jsonl/tree/v0.0.5...v0.0.5.1) +## [0.0.5.1](https://github.com/ome9ax/target-s3-jsonl/tree/0.0.5.1) (2021-08-12) +[Full Changelog](https://github.com/ome9ax/target-s3-jsonl/tree/0.0.5...0.0.5.1) ### Fixed bugs: - Issue to decompress archived files @@ -21,8 +30,8 @@ ### Merged pull requests: - [[compression] fix compression management](https://github.com/ome9ax/target-s3-jsonl/pull/12) -## [v0.0.5](https://github.com/ome9ax/target-s3-jsonl/tree/v0.0.5) (2021-08-12) -[Full Changelog](https://github.com/ome9ax/target-s3-jsonl/tree/v0.0.4...v0.0.5) +## [0.0.5](https://github.com/ome9ax/target-s3-jsonl/tree/0.0.5) (2021-08-12) +[Full Changelog](https://github.com/ome9ax/target-s3-jsonl/tree/0.0.4...0.0.5) ### New features: - I now store the rows in an Array on memory, and unload the Array into the file by batches. By default the batch size is 64Mb configurable with the `memory_buffer` config option. @@ -36,8 +45,8 @@ - [[File load buffer] unload the data from a 64Mb memory buffer](https://github.com/ome9ax/target-s3-jsonl/pull/8) - [[Metadata] manage tap Metadata _sdc columns according to the stitch documentation](https://github.com/ome9ax/target-s3-jsonl/pull/9) -## [v0.0.4](https://github.com/ome9ax/target-s3-jsonl/tree/v0.0.4) (2021-08-09) -[Full Changelog](https://github.com/ome9ax/target-s3-jsonl/tree/v0.0.0...v0.0.4) +## [0.0.4](https://github.com/ome9ax/target-s3-jsonl/tree/0.0.4) (2021-08-09) +[Full Changelog](https://github.com/ome9ax/target-s3-jsonl/tree/0.0.0...0.0.4) ### New features: - Initial release diff --git a/README.md b/README.md index 299ab3c..eeabb95 100644 --- a/README.md +++ b/README.md @@ -93,8 +93,8 @@ Full list of options in `config.json`: | s3_key_prefix | String | | (Default: None) A static prefix before the generated S3 key names. Using prefixes you can | encryption_type | String | | (Default: 'none') The type of encryption to use. Current supported options are: 'none' and 'KMS'. | | encryption_key | String | | A reference to the encryption key to use for data encryption. For KMS encryption, this should be the name of the KMS encryption key ID (e.g. '1234abcd-1234-1234-1234-1234abcd1234'). This field is ignored if 'encryption_type' is none or blank. | -| compression | String | | The type of compression to apply before uploading. Supported options are `none` (default), `gzip`, and `lzma`. For gzipped files, the file extension will automatically be changed to `.jsonl.gz` for all files. For `lzma` compression, the file extension will automatically be changed to `.jsonl.xz` for all files. | -| naming_convention | String | | (Default: None) Custom naming convention of the s3 key. Replaces tokens `date`, `stream`, and `timestamp` with the appropriate values.

Supports "folders" in s3 keys e.g. `folder/folder2/{stream}/export_date={date}/{timestamp}.jsonl`.

Honors the `s3_key_prefix`, if set, by prepending the "filename". E.g. naming_convention = `folder1/my_file.jsonl` and s3_key_prefix = `prefix_` results in `folder1/prefix_my_file.jsonl` | +| compression | String | | The type of compression to apply before uploading. Supported options are `none` (default), `gzip`, and `lzma`. For gzipped files, the file extension will automatically be changed to `.json.gz` for all files. For `lzma` compression, the file extension will automatically be changed to `.json.xz` for all files. | +| naming_convention | String | | (Default: None) Custom naming convention of the s3 key. Replaces tokens `date`, `stream`, and `timestamp` with the appropriate values.

Supports "folders" in s3 keys e.g. `folder/folder2/{stream}/export_date={date}/{timestamp}.json`.

Honors the `s3_key_prefix`, if set, by prepending the "filename". E.g. naming_convention = `folder1/my_file.json` and s3_key_prefix = `prefix_` results in `folder1/prefix_my_file.json` | | timezone_offset | Integer | | Use offset `0` hours is you want the `naming_convention` to use `utc` time zone. The `null` values is used by default. | | temp_dir | String | | (Default: platform-dependent) Directory of temporary JSONL files with RECORD messages. | | local | Boolean | | Keep the file in the `temp_dir` directory without uploading the files on `s3`. | diff --git a/setup.cfg b/setup.cfg index f78aa17..3230507 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,7 +49,7 @@ console_scripts = target-s3-jsonl = target_s3_jsonl:main [tool:pytest] -addopts = -v --cov=target_s3_jsonl --cov-fail-under 60 --cov-report annotate --cov-report xml --cov-report term --cov-report html:htmlcov --doctest-modules +addopts = -v --cov=target_s3_jsonl --cov-fail-under 95 --cov-report annotate --cov-report xml --cov-report term --cov-report html:htmlcov --doctest-modules testpaths = tests [coverage:run] diff --git a/target_s3_jsonl/__init__.py b/target_s3_jsonl/__init__.py index 6297718..0660426 100644 --- a/target_s3_jsonl/__init__.py +++ b/target_s3_jsonl/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -__version__ = '0.0.5.2' +__version__ = '0.0.6' import argparse import gzip @@ -181,10 +181,10 @@ def persist_lines(messages, config): raise message_type = o['type'] if message_type == 'RECORD': - if 'stream' not in o: # pragma: no cover + if 'stream' not in o: raise Exception("Line is missing required key 'stream': {}".format(message)) stream = o['stream'] - if stream not in schemas: # pragma: no cover + if stream not in schemas: raise Exception('A record for stream {} was encountered before a corresponding schema'.format(stream)) # NOTE: Validate record @@ -192,7 +192,8 @@ def persist_lines(messages, config): try: validators[stream].validate(float_to_decimal(record_to_load)) except Exception as ex: - if type(ex).__name__ == "InvalidOperation": # pragma: no cover + # NOTE: let anything but 'InvalidOperation' raised Exception slip by + if type(ex).__name__ == "InvalidOperation": # TODO pragma: no cover LOGGER.error( "Data validation failed and cannot load to destination. RECORD: {}\n" "'multipleOf' validations that allows long precisions are not supported" @@ -200,7 +201,7 @@ def persist_lines(messages, config): .format(record_to_load)) raise ex - if config.get('add_metadata_columns'): # pragma: no cover + if config.get('add_metadata_columns'): record_to_load = add_metadata_values_to_record(o, {}, now.timestamp()) else: record_to_load = remove_metadata_values_from_record(o) @@ -209,7 +210,7 @@ def persist_lines(messages, config): # NOTE: write temporary file # Use 64Mb default memory buffer - if sys.getsizeof(file_data[stream]['file_data']) > config.get('memory_buffer', 64e6): # pragma: no cover + if sys.getsizeof(file_data[stream]['file_data']) > config.get('memory_buffer', 64e6): save_file(file_data[stream], open_func) state = None @@ -217,38 +218,37 @@ def persist_lines(messages, config): LOGGER.debug('Setting state to {}'.format(o['value'])) state = o['value'] elif message_type == 'SCHEMA': - if 'stream' not in o: # pragma: no cover + if 'stream' not in o: raise Exception("Line is missing required key 'stream': {}".format(message)) stream = o['stream'] - if config.get('add_metadata_columns'): # pragma: no cover + if config.get('add_metadata_columns'): schemas[stream] = add_metadata_columns_to_schema(o) else: schemas[stream] = float_to_decimal(o['schema']) validators[stream] = Draft4Validator(schemas[stream], format_checker=FormatChecker()) - if 'key_properties' not in o: # pragma: no cover + if 'key_properties' not in o: raise Exception('key_properties field is required') key_properties[stream] = o['key_properties'] LOGGER.debug('Setting schema for {}'.format(stream)) # NOTE: get the s3 file key - if stream not in file_data: # pragma: no cover - file_data[stream] = { - 'target_key': get_target_key( - o, - naming_convention=naming_convention, - timestamp=now_formatted, - prefix=config.get('s3_key_prefix', ''), - timezone=timezone), - 'file_name': temp_dir / naming_convention_default.format(stream=stream, timestamp=now_formatted), - 'file_data': []} + file_data[stream] = { + 'target_key': get_target_key( + o, + naming_convention=naming_convention, + timestamp=now_formatted, + prefix=config.get('s3_key_prefix', ''), + timezone=timezone), + 'file_name': temp_dir / naming_convention_default.format(stream=stream, timestamp=now_formatted), + 'file_data': []} elif message_type == 'ACTIVATE_VERSION': LOGGER.debug('ACTIVATE_VERSION {}'.format(message)) - else: # pragma: no cover - LOGGER.warning('Unknown message type {} in message {}'.format(o['type'], o)) + else: + LOGGER.warning('Unknown message type "{}" in message "{}"'.format(o['type'], o)) for _, file_info in file_data.items(): save_file(file_info, open_func) diff --git a/target_s3_jsonl/s3.py b/target_s3_jsonl/s3.py index 16ca078..1384679 100644 --- a/target_s3_jsonl/s3.py +++ b/target_s3_jsonl/s3.py @@ -41,9 +41,9 @@ def create_client(config): ) # AWS Profile based authentication else: - aws_session = boto3.session.Session(profile_name=aws_profile) # pragma: no cover + aws_session = boto3.session.Session(profile_name=aws_profile) # TODO pragma: no cover if aws_endpoint_url: - s3 = aws_session.client('s3', endpoint_url=aws_endpoint_url) # pragma: no cover + s3 = aws_session.client('s3', endpoint_url=aws_endpoint_url) # TODO pragma: no cover else: s3 = aws_session.client('s3') return s3 @@ -59,7 +59,7 @@ def upload_file(filename, s3_client, bucket, s3_key, encryption_desc = "" encryption_args = None else: - if encryption_type.lower() == "kms": # pragma: no cover + if encryption_type.lower() == "kms": # TODO pragma: no cover encryption_args = {"ServerSideEncryption": "aws:kms"} if encryption_key: encryption_desc = ( @@ -70,7 +70,7 @@ def upload_file(filename, s3_client, bucket, s3_key, else: encryption_desc = " using default KMS encryption" else: - raise NotImplementedError( # pragma: no cover + raise NotImplementedError( "Encryption type '{}' is not supported. " "Expected: 'none' or 'KMS'" .format(encryption_type) diff --git a/tests/resources/config.json b/tests/resources/config.json index bbae8ad..9d54198 100644 --- a/tests/resources/config.json +++ b/tests/resources/config.json @@ -1,5 +1,4 @@ { - "local": true, "add_metadata_columns": false, "aws_access_key_id": "ACCESS-KEY", "aws_secret_access_key": "SECRET", diff --git a/tests/resources/config_local_false.json b/tests/resources/config_local.json similarity index 93% rename from tests/resources/config_local_false.json rename to tests/resources/config_local.json index f2b7b94..bbae8ad 100644 --- a/tests/resources/config_local_false.json +++ b/tests/resources/config_local.json @@ -1,5 +1,5 @@ { - "local": false, + "local": true, "add_metadata_columns": false, "aws_access_key_id": "ACCESS-KEY", "aws_secret_access_key": "SECRET", diff --git a/tests/resources/messages.json b/tests/resources/messages.json new file mode 100644 index 0000000..1fcb1a9 --- /dev/null +++ b/tests/resources/messages.json @@ -0,0 +1,6 @@ +{"type": "SCHEMA", "stream": "users", "key_properties": ["id"], "schema": {"required": ["id"], "type": "object", "properties": {"id": {"type": "integer"}}}} +{"type": "RECORD", "stream": "users", "record": {"id": 1, "name": "Eddy"}} +{"type": "RECORD", "stream": "users", "record": {"id": 2, "name": "Sabrina"}} +{"type": "SCHEMA", "stream": "locations", "key_properties": ["id"], "schema": {"required": ["id"], "type": "object", "properties": {"id": {"type": "integer"}}}} +{"type": "RECORD", "stream": "locations", "record": {"id": 1, "name": "Everywhere"}} +{"type": "STATE", "value": {"users": 2, "locations": 1}} diff --git a/tests/test_init.py b/tests/test_init.py index 9bc0168..0c16e5a 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -18,7 +18,6 @@ lzma, json, Path, - s3, add_metadata_columns_to_schema, add_metadata_values_to_record, remove_metadata_values_from_record, @@ -41,7 +40,7 @@ def utcnow(cls): return dt.fromtimestamp(1628663978.321056, tz=timezone.utc).replace(tzinfo=None) @classmethod - def now(cls, x=timezone.utc): + def now(cls, x=timezone.utc, tz=None): return cls.utcnow() @classmethod @@ -49,12 +48,33 @@ def utcfromtimestamp(cls, x): return cls.utcnow() @classmethod - def fromtimestamp(cls, x): + def fromtimestamp(cls, x, format): return cls.utcnow() + @classmethod + def strptime(cls, x, format): + return dt.strptime(x, format) + monkeypatch.setattr(datetime, 'datetime', mydatetime) +@fixture +def patch_argument_parser(monkeypatch): + + class argument_parser: + + def __init__(self): + self.config = str(Path('tests', 'resources', 'config.json')) + + def add_argument(self, x, y, help='Dummy config file', required=False): + pass + + def parse_args(self): + return self + + monkeypatch.setattr(argparse, 'ArgumentParser', argument_parser) + + @fixture def config(): '''Use custom parameters set''' @@ -277,7 +297,7 @@ def test_upload_files(monkeypatch, config, file_metadata): clear_dir(Path(config['temp_dir'])) -def test_persist_lines(config, input_data, invalid_row_data, invalid_order_data, state, file_metadata): +def test_persist_lines(caplog, config, input_data, invalid_row_data, invalid_order_data, state, file_metadata): '''TEST : simple persist_lines call''' output_state, output_file_metadata = persist_lines(input_data, config) file_paths = set(path for path in Path(config['temp_dir']).iterdir()) @@ -304,6 +324,28 @@ def test_persist_lines(config, input_data, invalid_row_data, invalid_order_data, clear_dir(Path(config['temp_dir'])) + config_copy = deepcopy(config) + config_copy['add_metadata_columns'] = True + output_state, output_file_metadata = persist_lines(input_data, config_copy) + + assert output_state == state + + clear_dir(Path(config['temp_dir'])) + + config_copy = deepcopy(config) + config_copy['memory_buffer'] = 9 + output_state, output_file_metadata = persist_lines(input_data, config_copy) + + assert output_state == state + + clear_dir(Path(config['temp_dir'])) + + dummy_type = '{"type": "DUMMY", "value": {"currently_syncing": "tap_dummy_test-test_table_one"}}' + output_state, output_file_metadata = persist_lines([dummy_type] + input_data, config) + + assert 'WARNING root:__init__.py:251 Unknown message type "{}" in message "{}"'.format( + json.loads(dummy_type)['type'], dummy_type.replace('"', "'")) + '\n' == caplog.text + with raises(NotImplementedError): config_copy = deepcopy(config) config_copy['compression'] = 'dummy' @@ -315,15 +357,67 @@ def test_persist_lines(config, input_data, invalid_row_data, invalid_order_data, with raises(Exception): output_state, output_file_metadata = persist_lines(invalid_order_data, config) + record = { + "type": "RECORD", + "stream": "tap_dummy_test-test_table_one", + "record": {"c_pk": 1, "c_varchar": "1", "c_int": 1}, + "version": 1, + "time_extracted": "2019-01-31T15:51:47.465408Z"} + + with raises(Exception): + dummy_input_data = deepcopy(input_data) + dummy_record = deepcopy(record) + dummy_record.pop('stream') + dummy_input_data.insert(3, json.dumps(dummy_record)) + output_state, output_file_metadata = persist_lines(dummy_input_data, config) + + schema = { + "type": "SCHEMA", + "stream": "tap_dummy_test-test_table_one", + "schema": { + "properties": { + "c_pk": {"inclusion": "automatic", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, + "c_varchar": {"inclusion": "available", "maxLength": 16, "type": ["null", "string"]}, + "c_int": {"inclusion": "available", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}}, + "type": "object"}, + "key_properties": ["c_pk"]} + + with raises(Exception): + dummy_input_data = deepcopy(input_data) + dummy_schema = deepcopy(schema) + dummy_schema.pop('stream') + dummy_input_data.insert(1, json.dumps(dummy_schema)) + output_state, output_file_metadata = persist_lines(dummy_input_data, config) + + with raises(Exception): + dummy_input_data = deepcopy(input_data) + dummy_schema = deepcopy(schema) + dummy_schema.pop('key_properties') + dummy_input_data.insert(1, json.dumps(dummy_schema)) + output_state, output_file_metadata = persist_lines(dummy_input_data, config) + @mock_s3 -def test_main(monkeypatch, capsys, patch_datetime, input_data, config, state, file_metadata): +def test_main(monkeypatch, capsys, patch_datetime, patch_argument_parser, input_data, config, state, file_metadata): '''TEST : simple persist_lines call''' + monkeypatch.setattr(sys, 'stdin', input_data) + + conn = boto3.resource('s3', region_name='us-east-1') + conn.create_bucket(Bucket=config['s3_bucket']) + + main() + + captured = capsys.readouterr() + assert captured.out == json.dumps(state) + '\n' + + for _, file_info in file_metadata.items(): + assert not file_info['file_name'].exists() + class argument_parser: def __init__(self): - self.config = str(Path('tests', 'resources', 'config.json')) + self.config = str(Path('tests', 'resources', 'config_local.json')) def add_argument(self, x, y, help='Dummy config file', required=False): pass @@ -333,9 +427,8 @@ def parse_args(self): monkeypatch.setattr(argparse, 'ArgumentParser', argument_parser) - monkeypatch.setattr(sys, 'stdin', input_data) - main() + captured = capsys.readouterr() assert captured.out == json.dumps(state) + '\n' @@ -360,26 +453,3 @@ def parse_args(self): monkeypatch.setattr(argparse, 'ArgumentParser', argument_parser) main() - - class argument_parser: - - def __init__(self): - self.config = str(Path('tests', 'resources', 'config_local_false.json')) - - def add_argument(self, x, y, help='Dummy config file', required=False): - pass - - def parse_args(self): - return self - - monkeypatch.setattr(argparse, 'ArgumentParser', argument_parser) - - monkeypatch.setattr(s3, 'create_client', lambda config: None) - - monkeypatch.setattr( - s3, 'upload_file', lambda filename, s3_client, bucket, s3_key, - encryption_type=None, encryption_key=None: None) - - main() - captured = capsys.readouterr() - assert captured.out == json.dumps(state) + '\n' diff --git a/tests/test_s3.py b/tests/test_s3.py new file mode 100644 index 0000000..a30d257 --- /dev/null +++ b/tests/test_s3.py @@ -0,0 +1,93 @@ +'''Tests for the target_s3_jsonl.main module''' +# Standard library imports +from copy import deepcopy +from pathlib import Path +import json + +# Third party imports +from pytest import fixture, raises +import boto3 +from botocore.errorfactory import ClientError +from moto import mock_s3 + +# Package imports +from target_s3_jsonl.s3 import create_client, upload_file + + +@fixture +def config(): + '''Use custom parameters set''' + + with open(Path('tests', 'resources', 'config.json'), 'r', encoding='utf-8') as config_file: + return json.load(config_file) + + +@mock_s3 +def test_create_client(config): + '''TEST : simple upload_files call''' + + conn = boto3.resource('s3', region_name='us-east-1') + # We need to create the bucket since this is all in Moto's 'virtual' AWS account + conn.create_bucket(Bucket=config['s3_bucket']) + + client = create_client(config) + client.put_object(Bucket=config['s3_bucket'], Key='Eddy is', Body='awesome!') + body = conn.Object(config['s3_bucket'], 'Eddy is').get()[ + 'Body'].read().decode("utf-8") + + assert body == 'awesome!' + + with raises(Exception): + config_copy = deepcopy(config) + config_copy['aws_endpoint_url'] = 'xx' + + client = create_client(config_copy) + client.put_object(Bucket=config_copy['s3_bucket'], Key='Eddy is', Body='awesome!') + body = conn.Object(config_copy['s3_bucket'], 'Eddy is').get()[ + 'Body'].read().decode("utf-8") + + +@mock_s3 +def test_upload_file(config): + '''TEST : simple upload_files call''' + + conn = boto3.resource('s3', region_name='us-east-1') + conn.create_bucket(Bucket=config['s3_bucket']) + + client = create_client(config) + + file_key = str(Path('tests', 'resources', 'config.json')) + upload_file( + file_key, + client, + config.get('s3_bucket'), + 'dummy/remote_config.json', + encryption_type=config.get('encryption_type'), + encryption_key=config.get('encryption_key')) + + try: + client.head_object(Bucket=config.get('s3_bucket'), Key=file_key) + except ClientError: + pass + + with raises(Exception): + upload_file( + file_key, + client, + config.get('s3_bucket'), + 'dummy/remote_config_dummy.json', + encryption_type='dummy', + encryption_key=config.get('encryption_key')) + + upload_file( + file_key, + client, + config.get('s3_bucket'), + 'dummy/remote_config_kms.json', + encryption_type='kms', + encryption_key=config.get('encryption_key')) + + try: + client.head_object(Bucket=config.get('s3_bucket'), Key=file_key) + except ClientError: + pass