diff --git a/TrafficCapture/README.md b/TrafficCapture/README.md index 93f4e8e89..1b424572e 100644 --- a/TrafficCapture/README.md +++ b/TrafficCapture/README.md @@ -110,27 +110,46 @@ The body of the messages is sometimes gzipped which makes it difficult to repres and responses is base64 encoded before it is logged. This makes the files stable, but not human-readable. We have provided a utility script that can parse these files and output them to a human-readable format: the bodies are -base64 decoded, un-gzipped if applicable, and parsed as JSON if applicable. They're then saved back to JSON format on disk. +base64 decoded and parsed as JSON if applicable. They're then saved back to JSON format to stdout or file. To use this utility from the Migration Console, ```sh -$ ./humanReadableLogs.py --help -usage: humanReadableLogs.py [-h] [--outfile OUTFILE] infile - -positional arguments: - infile Path to input logged tuple file. - -options: - -h, --help show this help message and exit - --outfile OUTFILE Path for output human readable tuple file. - -# By default, the output file is the same path as the input file, but the file name is prefixed with `readable-`. -$ ./humanReadableLogs.py /shared_replayer_output/tuples.log -Input file: /shared_replayer_output/tuples.log; Output file: /shared_replayer_output/readable-tuples.log - +$ console tuples show --help +Usage: console tuples convert [OPTIONS] + +Options: + --in FILENAME + --out FILENAME + --help Show this message and exit. + +# By default, the input and output files are `stdin` and `stdout` respectively, so they can be piped together with other tools. +$ console tuples show --in /shared-logs-output/traffic-replayer-default/86ca83e66197/tuples/mini_tuples.log | jq +{ + "sourceRequest": { + "Request-URI": "/", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "User-Agent": "python-requests/2.32.3", + "Accept-Encoding": "gzip, deflate, zstd", + "Accept": "*/*", + "Connection": "keep-alive", + "Authorization": "Basic YWRtaW46YWRtaW4=", + "body": "" + }, + "sourceResponse": { + "HTTP-Version": { + "keepAliveDefault": true + }, + "Status-Code": 200, + "Reason-Phrase": "OK", + ... + }, + ... +} # A specific output file can also be specified. -$ ./humanReadableLogs.py /shared_replayer_output/tuples.log --outfile local-tuples.log -Input file: /shared_replayer_output/tuples.log; Output file: local-tuples.log +$ console tuples show --in /shared_replayer_output/tuples.log --out local-tuples.log +Converted tuples output to local-tuples.log ``` ### Capture Kafka Offloader diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile index d2c5b2c9e..ab02505f2 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile @@ -27,9 +27,6 @@ COPY osiPipelineTemplate.yaml /root/ COPY msk-iam-auth.properties /root/kafka-tools/aws COPY kafkaCmdRef.md /root/kafka-tools -COPY humanReadableLogs.py /root/ -RUN chmod ug+x /root/humanReadableLogs.py - COPY showFetchMigrationCommand.sh /root/ RUN chmod ug+x /root/showFetchMigrationCommand.sh @@ -57,7 +54,7 @@ RUN echo '. /.venv/bin/activate' >> /etc/profile.d/venv.sh RUN dnf install -y bash-completion RUN echo '. /etc/profile.d/bash_completion.sh' >> ~/.bashrc && \ echo '. /etc/profile.d/venv.sh' >> ~/.bashrc && \ - echo '@echo Welcome to the Migration Assistant Console' >> ~/.bashrc + echo 'echo Welcome to the Migration Assistant Console' >> ~/.bashrc # Set ENV to control startup script in /bin/sh mode ENV ENV=/root/.bashrc diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/humanReadableLogs.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/humanReadableLogs.py deleted file mode 100755 index e6c1a3130..000000000 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/humanReadableLogs.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import base64 -import gzip -import json -import pathlib -from typing import Optional -import logging - -from tqdm import tqdm -from tqdm.contrib.logging import logging_redirect_tqdm - -logger = logging.getLogger(__name__) - -BASE64_ENCODED_TUPLE_PATHS = ["sourceRequest.body", "targetRequest.body", "sourceResponse.body"] -# TODO: I'm not positive about the capitalization of the Content-Encoding and Content-Type headers. -# This version worked on my test cases, but not guaranteed to work in all cases. -CONTENT_ENCODING_PATH = { - BASE64_ENCODED_TUPLE_PATHS[0]: "sourceRequest.Content-Encoding", - BASE64_ENCODED_TUPLE_PATHS[1]: "targetRequest.Content-Encoding", - BASE64_ENCODED_TUPLE_PATHS[2]: "sourceResponse.Content-Encoding" -} -CONTENT_TYPE_PATH = { - BASE64_ENCODED_TUPLE_PATHS[0]: "sourceRequest.Content-Type", - BASE64_ENCODED_TUPLE_PATHS[1]: "targetRequest.Content-Type", - BASE64_ENCODED_TUPLE_PATHS[2]: "sourceResponse.Content-Type" -} -TRANSFER_ENCODING_PATH = { - BASE64_ENCODED_TUPLE_PATHS[0]: "sourceRequest.Transfer-Encoding", - BASE64_ENCODED_TUPLE_PATHS[1]: "targetRequest.Transfer-Encoding", - BASE64_ENCODED_TUPLE_PATHS[2]: "sourceResponse.Transfer-Encoding" -} - -CONTENT_TYPE_JSON = "application/json" -CONTENT_ENCODING_GZIP = "gzip" -TRANSFER_ENCODING_CHUNKED = "chunked" -URI_PATH = "sourceRequest.Request-URI" -BULK_URI_PATH = "_bulk" - - -class DictionaryPathException(Exception): - pass - - -def get_element(element: str, dict_: dict, raise_on_error=False, try_lowercase_keys=False) -> Optional[any]: - """This has a limited version of case-insensitivity. It specifically only checks the provided key - and an all lower-case version of the key (if `try_lowercase_keys` is True).""" - keys = element.split('.') - rv = dict_ - for key in keys: - try: - if key in rv: - rv = rv[key] - continue - if try_lowercase_keys and key.lower() in rv: - rv = rv[key.lower()] - except KeyError: - if raise_on_error: - raise DictionaryPathException(f"Key {key} was not present.") - else: - return None - return rv - - -def set_element(element: str, dict_: dict, value: any) -> None: - keys = element.split('.') - rv = dict_ - for key in keys[:-1]: - rv = rv[key] - rv[keys[-1]] = value - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("infile", type=pathlib.Path, help="Path to input logged tuple file.") - parser.add_argument("--outfile", type=pathlib.Path, help="Path for output human readable tuple file.") - return parser.parse_args() - - -def decode_chunked(data: bytes) -> bytes: - newdata = [] - next_newline = data.index(b'\r\n') - chunk = data[next_newline + 2:] - while len(chunk) > 7: # the final EOM chunk is 7 bytes - next_newline = chunk.index(b'\r\n') - newdata.append(chunk[:next_newline]) - chunk = chunk[next_newline + 2:] - return b''.join(newdata) - - -def parse_body_value(raw_value: str, content_encoding: Optional[str], - content_type: Optional[str], is_bulk: bool, is_chunked_transfer: bool, line_no: int): - # Body is base64 decoded - try: - b64decoded = base64.b64decode(raw_value) - except Exception as e: - logger.error(f"Body value on line {line_no} could not be decoded: {e}. Skipping parsing body value.") - return None - - # Decoded data is un-chunked, if applicable - if is_chunked_transfer: - contiguous_data = decode_chunked(b64decoded) - else: - contiguous_data = b64decoded - - # Data is un-gzipped, if applicable - is_gzipped = content_encoding is not None and content_encoding == CONTENT_ENCODING_GZIP - if is_gzipped: - try: - unzipped = gzip.decompress(contiguous_data) - except Exception as e: - logger.error(f"Body value on line {line_no} should be gzipped but could not be unzipped: {e}. " - "Skipping parsing body value.") - return contiguous_data - else: - unzipped = contiguous_data - - # Data is decoded to utf-8 string - try: - decoded = unzipped.decode("utf-8") - except Exception as e: - logger.error(f"Body value on line {line_no} could not be decoded to utf-8: {e}. " - "Skipping parsing body value.") - return unzipped - - # Data is parsed as json, if applicable - is_json = content_type is not None and CONTENT_TYPE_JSON in content_type - if is_json and len(decoded) > 0: - # Data is parsed as a bulk json, if applicable - if is_bulk: - try: - return [json.loads(line) for line in decoded.splitlines()] - except Exception as e: - logger.error("Body value on line {line_no} should be a bulk json (list of json lines) but " - f"could not be parsed: {e}. Skipping parsing body value.") - return decoded - try: - return json.loads(decoded) - except Exception as e: - logger.error(f"Body value on line {line_no} should be a json but could not be parsed: {e}. " - "Skipping parsing body value.") - return decoded - return decoded - - -def parse_tuple(line: str, line_no: int) -> dict: - tuple = json.loads(line) - try: - is_bulk_path = BULK_URI_PATH in get_element(URI_PATH, tuple, raise_on_error=True) - except DictionaryPathException as e: - logger.error(f"`{URI_PATH}` on line {line_no} could not be loaded: {e} " - f"Skipping parsing tuple.") - return tuple - for body_path in BASE64_ENCODED_TUPLE_PATHS: - base64value = get_element(body_path, tuple) - if base64value is None: - # This component has no body element, which is potentially valid. - continue - value = decode_base64_http_message(base64value, CONTENT_ENCODING_PATH[body_path], CONTENT_TYPE_PATH[body_path], - TRANSFER_ENCODING_PATH[body_path], is_bulk_path, line_no, tuple) - if value and type(value) is not bytes: - set_element(body_path, tuple, value) - for target_response in get_element("targetResponses", tuple): - value = decode_base64_http_message(base64value, "Content-Encoding", "Content-Type", - "Transfer-Encoding", is_bulk_path, line_no, target_response) - if value and type(value) is not bytes: - set_element("body", target_response, value) - return tuple - - -def decode_base64_http_message(base64value, content_encoding, content_type, transfer_encoding, - is_bulk_path, line_no, tuple): - content_encoding = get_element(content_encoding, tuple, try_lowercase_keys=True) - content_type = get_element(content_type, tuple, try_lowercase_keys=True) - is_chunked_transfer = get_element(transfer_encoding, - tuple, try_lowercase_keys=True) == TRANSFER_ENCODING_CHUNKED - return parse_body_value(base64value, content_encoding, content_type, is_bulk_path, - is_chunked_transfer, line_no) - - -if __name__ == "__main__": - args = parse_args() - if args.outfile: - outfile = args.outfile - else: - outfile = args.infile.parent / f"readable-{args.infile.name}" - print(f"Input file: {args.infile}; Output file: {outfile}") - - logging.basicConfig(level=logging.INFO) - with logging_redirect_tqdm(): - with open(args.infile, 'r') as in_f: - with open(outfile, 'w') as out_f: - for i, line in tqdm(enumerate(in_f)): - print(json.dumps(parse_tuple(line, i + 1)), file=out_f) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py index c7f5b0d01..aa9a9544a 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py @@ -1,5 +1,6 @@ import json from pprint import pprint +import sys import click import console_link.middleware.clusters as clusters_ import console_link.middleware.metrics as metrics_ @@ -8,6 +9,7 @@ import console_link.middleware.metadata as metadata_ import console_link.middleware.replay as replay_ import console_link.middleware.kafka as kafka_ +import console_link.middleware.tuples as tuples_ from console_link.models.utils import ExitCode from console_link.environment import Environment @@ -485,6 +487,26 @@ def completion(ctx, config_file, json, shell): ctx.exit(1) +@cli.group(name="tuples") +@click.pass_obj +def tuples_group(ctx): + """ All commands related to tuples. """ + pass + + +@tuples_group.command() +@click.option('--in', 'inputfile', + type=click.File('r'), + default=sys.stdin) +@click.option('--out', 'outputfile', + type=click.File('a'), + default=sys.stdout) +def show(inputfile, outputfile): + tuples_.convert(inputfile, outputfile) + if outputfile != sys.stdout: + click.echo(f"Converted tuples output to {outputfile.name}") + + ################################################# if __name__ == "__main__": diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/middleware/tuples.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/middleware/tuples.py new file mode 100644 index 000000000..ba8363c34 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/middleware/tuples.py @@ -0,0 +1,10 @@ +import logging +import typing + +from console_link.models.tuple_reader import TupleReader +logger = logging.getLogger(__name__) + + +def convert(inputfile: typing.TextIO, ouptutfile: typing.TextIO): + tuple_reader = TupleReader() + tuple_reader.transform_stream(inputfile, ouptutfile) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/tuple_reader.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/tuple_reader.py new file mode 100644 index 000000000..f055c5c6b --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/tuple_reader.py @@ -0,0 +1,214 @@ +from enum import Enum +import re + +import json +from typing import Any, Dict, Generator, List, Self, Set, TextIO, Union +import base64 +from typing import Optional + +import logging + +logger = logging.getLogger(__name__) + + +class TupleReader: + """ This class is fairly minimal for now. There is likely a future in which multiple + tuple storage locations/types are supported, but we are not there yet and don't have + a clear enough vision for it to make sense to frame it out now.""" + def __init__(self) -> None: + # Initialize a TupleReader object. + pass + + def transform_stream(self, inputfile: TextIO, outputfile: TextIO): + transformer = self._transform_lines(inputfile.readlines()) + while True: + try: + json.dump(next(transformer), outputfile) + outputfile.write('\n') + except StopIteration: + logger.info("Reached the end of the input object") + return + + def _transform_lines(self, lines: List[str]) -> Generator[Dict, None, None]: + for i, line in enumerate(lines): + yield parse_tuple(line, i + 1) + + +CONTENT_TYPE_JSON = "application/json" +BULK_URI_PATH = "_bulk" + +SOURCE_REQUEST = "sourceRequest" +TARGET_REQUEST = "targetRequest" +SOURCE_RESPONSE = "sourceResponse" +TARGET_RESPONSE = "targetResponses" + +SINGLE_COMPONENTS = [SOURCE_REQUEST, SOURCE_RESPONSE, TARGET_REQUEST] +LIST_COMPONENTS = [TARGET_RESPONSE] + +URI_PATH = SOURCE_REQUEST + ".Request-URI" + +CONTENT_TYPE_REGEX = re.compile('Content-Type', flags=re.IGNORECASE) + + +class DictionaryPathException(Exception): + pass + + +def get_element_with_regex(regex: re.Pattern, dict_: Dict, raise_on_error=False): + keys = dict_.keys() + try: + match = next(filter(regex.match, keys)) + except StopIteration: + if raise_on_error: + raise DictionaryPathException(f"An element matching the regex ({regex}) was not found.") + return None + + return dict_[match] + + +def get_element(element: str, dict_: dict, raise_on_error=False, try_lowercase_keys=False) -> Optional[any]: + """This has a limited version of case-insensitivity. It specifically only checks the provided key + and an all lower-case version of the key (if `try_lowercase_keys` is True).""" + keys = element.split('.') + rv = dict_ + for key in keys: + try: + if key in rv: + rv = rv[key] + continue + elif try_lowercase_keys and key.lower() in rv: + rv = rv[key.lower()] + else: + raise KeyError + except KeyError: + if raise_on_error: + raise DictionaryPathException(f"Key {key} was not present.") + else: + return None + return rv + + +def set_element(element: str, dict_: dict, value: any) -> None: + keys = element.split('.') + rv = dict_ + for key in keys[:-1]: + try: + rv = rv[key] + except KeyError: + raise DictionaryPathException(f"Key {key} was not present.") + try: + rv[keys[-1]] = value + except TypeError: + raise DictionaryPathException(f"Path {element} did not reach an assignable object.") + + +Flag = Enum('Flag', ['Bulk_Request', 'Json']) + + +class TupleComponent: + def __init__(self, component_name: str, component: Dict, line_no: int, is_bulk_path: bool): + body = get_element("body", component) + self.value: Union[bytes, str] = body + + self.flags = get_flags_for_component(component, is_bulk_path) + + self.line_no = line_no + self.component_name = component_name + + self.final_value: Union[Dict, List, str, bytes] = {} + self.error = False + + def b64decode(self) -> Self: + if self.error or self.value is None: + return self + try: + self.value = base64.b64decode(self.value) + except Exception as e: + self.error = (f"Body value of {self.component_name} on line {self.line_no} could not be decoded: {e}." + "Skipping parsing body value.") + logger.debug(self.error) + logger.debug(self.value) + return self + + def decode_utf8(self) -> Self: + if self.error or self.value is None: + return self + try: + self.value = self.value.decode("utf-8") + except Exception as e: + self.error = (f"Body value of {self.component_name} on line {self.line_no} could not be decoded to utf-8: " + f"{e}. Skipping parsing body value.") + logger.debug(self.error) + logger.debug(self.value) + return self + + def parse_as_json(self) -> Self: + if self.error or self.value is None: + return self + if Flag.Json not in self.flags: + self.final_value = self.value + return self + + if self.value.strip() == "": + self.final_value = self.value + return self + + if Flag.Bulk_Request in self.flags: + try: + self.final_value = [json.loads(line) for line in self.value.splitlines()] + except Exception as e: + self.error = (f"Body value of {self.component_name} on line {self.line_no} should be a bulk json, but " + f"could not be parsed: {e}. Skipping parsing body value.") + logger.debug(self.error) + logger.debug(self.value) + self.final_value = self.value + else: + try: + self.final_value = json.loads(self.value) + except Exception as e: + self.error = (f"Body value of {self.component_name} on line {self.line_no} should be a json, but " + f"could not be parsed: {e}. Skipping parsing body value.") + logger.debug(self.error) + logger.debug(self.value) + self.final_value = self.value + return self + + +def get_flags_for_component(component: Dict[str, Any], is_bulk_path: bool) -> Set[Flag]: + content_type = get_element_with_regex(CONTENT_TYPE_REGEX, component) + is_json = content_type is not None and CONTENT_TYPE_JSON in content_type + return {Flag.Json if is_json else None, + Flag.Bulk_Request if is_bulk_path else None} - {None} + + +def parse_tuple(line: str, line_no: int) -> dict: + initial_tuple = json.loads(line) + try: + is_bulk_path = BULK_URI_PATH in get_element(URI_PATH, initial_tuple, raise_on_error=True) + except DictionaryPathException as e: + logger.error(f"`{URI_PATH}` on line {line_no} could not be loaded: {e} " + f"Skipping parsing tuple.") + return initial_tuple + + for component in SINGLE_COMPONENTS: + tuple_component = TupleComponent(component, initial_tuple[component], line_no, is_bulk_path) + + processed_tuple = tuple_component.b64decode().decode_utf8().parse_as_json() + final_value = processed_tuple.final_value + if not processed_tuple.error: + set_element(component + ".body", initial_tuple, final_value) + else: + logger.error(processed_tuple.error) + + for component in LIST_COMPONENTS: + for i, item in enumerate(initial_tuple[component]): + tuple_component = TupleComponent(f"{component} item {i}", item, line_no, is_bulk_path) + + processed_tuple = tuple_component.b64decode().decode_utf8().parse_as_json() + final_value = processed_tuple.final_value + if not processed_tuple.error: + set_element("body", item, final_value) + else: + logger.error(processed_tuple.error) + + return initial_tuple diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/invalid_tuple.json b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/invalid_tuple.json new file mode 100644 index 000000000..7960fdfda --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/invalid_tuple.json @@ -0,0 +1,51 @@ +{ + "sourceRequest": { + "Request-URI": "/_cat/indices?v", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "Authorization": "Basic YWRtaW46YWRtaW4=", + "User-Agent": "curl/8.5.0", + "Accept": "*/*", + "body": "" + }, + "sourceResponse": { + "HTTP-Version": { + "keepAliveDefault": true + }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 59, + "content-type": "text/plain; charset=UTF-8", + "content-length": "214", + "comment": "This body has a few characters modified to make it un-decodeable as utf-8", + "body": "aGVhbHRoIHN0YXR1cyBpbmRleCAgICACB1dWlkICAgICAgICAgICAgICAgICAgIHByaSByZXAgZG9jcy5jb3VudCBkb2NzLmRlbGV0ZWQgc3RvcmUuc2l6ZSBwcmkuc3RvcmUuc2l6ZQpncmVlbiAgb3BlbiAgIHNlYXJjaGd1YXJkIHlKS1hQTUh0VFJPTklYU1pYQ193bVEgICAxICAgMCAgICAgICAgICA4ICAgICAgICAgICAgMCAgICAgNDQuN2tiICAgICAgICAgNDQuN2tiCg==" + }, + "targetRequest": { + "Request-URI": "/_cat/indices?v", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "Authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "User-Agent": "curl/8.5.0", + "Accept": "*/*", + "body": "" + }, + "targetResponses": [ + { + "HTTP-Version": { + "keepAliveDefault": true + }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 721, + "content-type": "application/json; charset=UTF-8", + "content-length": "484", + "comment": "This response is valid text, but is not valid as json, despite the `content-type` header", + "body": "aGVhbHRoIHN0YXR1cyBpbmRleCAgICAgICAgICAgICAgICAgICAgIHV1aWQgICAgICAgICAgICAgICAgICAgcHJpIHJlcCBkb2NzLmNvdW50IGRvY3MuZGVsZXRlZCBzdG9yZS5zaXplIHByaS5zdG9yZS5zaXplCmdyZWVuICBvcGVuICAgLm9wZW5zZWFyY2gtb2JzZXJ2YWJpbGl0eSA4Vy1vWUhmYlN5U3JkeFFFX3NPbnpnICAgMSAgIDAgICAgICAgICAgMCAgICAgICAgICAgIDAgICAgICAgMjA4YiAgICAgICAgICAgMjA4YgpncmVlbiAgb3BlbiAgIC5wbHVnaW5zLW1sLWNvbmZpZyAgICAgICAgRjludnh2c2dSelNibG1mSnZ2aGptdyAgIDEgICAwICAgICAgICAgIDEgICAgICAgICAgICAwICAgICAgMy44a2IgICAgICAgICAgMy44a2IKZ3JlZW4gIG9wZW4gICAub3BlbmRpc3Ryb19zZWN1cml0eSAgICAgIDVmWHlhbkZuU2tDUUQ2bjFKUW1KTlEgICAxICAgMCAgICAgICAgIDEwICAgICAgICAgICAgMCAgICAgNzcuNWtiICAgICAgICAgNzcuNWtiCg==" + } + ], + "connectionId": "0242acfffe13000a-0000000a-00000005-1eb087a9beb83f3e-a32794b4.0", + "numRequests": 1, + "numErrors": 0 +} diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/multiple_tuples.json b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/multiple_tuples.json new file mode 100644 index 000000000..d0f7eec8a --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/multiple_tuples.json @@ -0,0 +1,208 @@ +[ + { + "sourceRequest": { + "Request-URI": "/", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "User-Agent": "python-requests/2.32.3", + "Accept-Encoding": "gzip, deflate, zstd", + "Accept": "*/*", + "Connection": "keep-alive", + "Authorization": "Basic YWRtaW46YWRtaW4=", + "body": "" + }, + "sourceResponse": { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 374, + "content-type": "application/json; charset=UTF-8", + "content-length": "538", + "body": "ewogICJuYW1lIiA6ICIzY2MwNjhhZDU0ZWIiLAogICJjbHVzdGVyX25hbWUiIDogImRvY2tlci1jbHVzdGVyIiwKICAiY2x1c3Rlcl91dWlkIiA6ICJQSUFhd0R3U1FLU1JMdG9hWEMtb1dnIiwKICAidmVyc2lvbiIgOiB7CiAgICAibnVtYmVyIiA6ICI3LjEwLjIiLAogICAgImJ1aWxkX2ZsYXZvciIgOiAib3NzIiwKICAgICJidWlsZF90eXBlIiA6ICJkb2NrZXIiLAogICAgImJ1aWxkX2hhc2giIDogIjc0N2UxY2M3MWRlZjA3NzI1Mzg3OGE1OTE0M2MxZjc4NWFmYTkyYjkiLAogICAgImJ1aWxkX2RhdGUiIDogIjIwMjEtMDEtMTNUMDA6NDI6MTIuNDM1MzI2WiIsCiAgICAiYnVpbGRfc25hcHNob3QiIDogZmFsc2UsCiAgICAibHVjZW5lX3ZlcnNpb24iIDogIjguNy4wIiwKICAgICJtaW5pbXVtX3dpcmVfY29tcGF0aWJpbGl0eV92ZXJzaW9uIiA6ICI2LjguMCIsCiAgICAibWluaW11bV9pbmRleF9jb21wYXRpYmlsaXR5X3ZlcnNpb24iIDogIjYuMC4wLWJldGExIgogIH0sCiAgInRhZ2xpbmUiIDogIllvdSBLbm93LCBmb3IgU2VhcmNoIgp9Cg==" + }, + "targetRequest": { + "Request-URI": "/", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "User-Agent": "python-requests/2.32.3", + "Accept-Encoding": "gzip, deflate, zstd", + "Accept": "*/*", + "Connection": "keep-alive", + "Authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "body": "" + }, + "targetResponses": [ + { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 341, + "content-type": "application/json; charset=UTF-8", + "content-length": "568", + "body": "ewogICJuYW1lIiA6ICJlNmIzYTQzNzA3NjMiLAogICJjbHVzdGVyX25hbWUiIDogImRvY2tlci1jbHVzdGVyIiwKICAiY2x1c3Rlcl91dWlkIiA6ICI4OHpsaWs3T1JGdUZUNzh6ZHVyYWRBIiwKICAidmVyc2lvbiIgOiB7CiAgICAiZGlzdHJpYnV0aW9uIiA6ICJvcGVuc2VhcmNoIiwKICAgICJudW1iZXIiIDogIjIuMTUuMCIsCiAgICAiYnVpbGRfdHlwZSIgOiAidGFyIiwKICAgICJidWlsZF9oYXNoIiA6ICI2MWRiY2QwNzk1YzliZmU5YjgxZTU3NjIxNzU0MTRiYzM4YmJjYWRmIiwKICAgICJidWlsZF9kYXRlIiA6ICIyMDI0LTA2LTIwVDAzOjI3OjMyLjU2MjAzNjg5MFoiLAogICAgImJ1aWxkX3NuYXBzaG90IiA6IGZhbHNlLAogICAgImx1Y2VuZV92ZXJzaW9uIiA6ICI5LjEwLjAiLAogICAgIm1pbmltdW1fd2lyZV9jb21wYXRpYmlsaXR5X3ZlcnNpb24iIDogIjcuMTAuMCIsCiAgICAibWluaW11bV9pbmRleF9jb21wYXRpYmlsaXR5X3ZlcnNpb24iIDogIjcuMC4wIgogIH0sCiAgInRhZ2xpbmUiIDogIlRoZSBPcGVuU2VhcmNoIFByb2plY3Q6IGh0dHBzOi8vb3BlbnNlYXJjaC5vcmcvIgp9Cg==" + } + ], + "connectionId": "0242acfffe12000b-0000000a-00000003-4c7bb8f4aefa3189-c0544dbf.0", + "numRequests": 1, + "numErrors": 0 + }, + { + "sourceRequest": { + "Request-URI": "/geonames", + "Method": "PUT", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46YWRtaW4=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "1214", + "body": "eyJzZXR0aW5ncyI6eyJpbmRleC5udW1iZXJfb2Zfc2hhcmRzIjo1LCJpbmRleC5udW1iZXJfb2ZfcmVwbGljYXMiOjAsImluZGV4LnN0b3JlLnR5cGUiOiJmcyIsImluZGV4LnF1ZXJpZXMuY2FjaGUuZW5hYmxlZCI6ZmFsc2UsImluZGV4LnJlcXVlc3RzLmNhY2hlLmVuYWJsZSI6ZmFsc2V9LCJtYXBwaW5ncyI6eyJkeW5hbWljIjoic3RyaWN0IiwiX3NvdXJjZSI6eyJlbmFibGVkIjp0cnVlfSwicHJvcGVydGllcyI6eyJlbGV2YXRpb24iOnsidHlwZSI6ImludGVnZXIifSwibmFtZSI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJnZW9uYW1laWQiOnsidHlwZSI6ImxvbmcifSwiZmVhdHVyZV9jbGFzcyI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJsb2NhdGlvbiI6eyJ0eXBlIjoiZ2VvX3BvaW50In0sImNjMiI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJ0aW1lem9uZSI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJkZW0iOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiY291bnRyeV9jb2RlIjp7InR5cGUiOiJ0ZXh0IiwiZmllbGRkYXRhIjp0cnVlLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiYWRtaW4xX2NvZGUiOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiYWRtaW4yX2NvZGUiOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiYWRtaW4zX2NvZGUiOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiYWRtaW40X2NvZGUiOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiZmVhdHVyZV9jb2RlIjp7InR5cGUiOiJ0ZXh0IiwiZmllbGRzIjp7InJhdyI6eyJ0eXBlIjoia2V5d29yZCJ9fX0sImFsdGVybmF0ZW5hbWVzIjp7InR5cGUiOiJ0ZXh0IiwiZmllbGRzIjp7InJhdyI6eyJ0eXBlIjoia2V5d29yZCJ9fX0sImFzY2lpbmFtZSI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJwb3B1bGF0aW9uIjp7InR5cGUiOiJsb25nIn19fX0=" + }, + "sourceResponse": { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 232, + "content-type": "application/json; charset=UTF-8", + "content-length": "67", + "body": "eyJhY2tub3dsZWRnZWQiOnRydWUsInNoYXJkc19hY2tub3dsZWRnZWQiOnRydWUsImluZGV4IjoiZ2VvbmFtZXMifQ==" + }, + "targetRequest": { + "Request-URI": "/geonames", + "Method": "PUT", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "1214", + "body": "eyJzZXR0aW5ncyI6eyJpbmRleC5udW1iZXJfb2Zfc2hhcmRzIjo1LCJpbmRleC5udW1iZXJfb2ZfcmVwbGljYXMiOjAsImluZGV4LnN0b3JlLnR5cGUiOiJmcyIsImluZGV4LnF1ZXJpZXMuY2FjaGUuZW5hYmxlZCI6ZmFsc2UsImluZGV4LnJlcXVlc3RzLmNhY2hlLmVuYWJsZSI6ZmFsc2V9LCJtYXBwaW5ncyI6eyJkeW5hbWljIjoic3RyaWN0IiwiX3NvdXJjZSI6eyJlbmFibGVkIjp0cnVlfSwicHJvcGVydGllcyI6eyJlbGV2YXRpb24iOnsidHlwZSI6ImludGVnZXIifSwibmFtZSI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJnZW9uYW1laWQiOnsidHlwZSI6ImxvbmcifSwiZmVhdHVyZV9jbGFzcyI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJsb2NhdGlvbiI6eyJ0eXBlIjoiZ2VvX3BvaW50In0sImNjMiI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJ0aW1lem9uZSI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJkZW0iOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiY291bnRyeV9jb2RlIjp7InR5cGUiOiJ0ZXh0IiwiZmllbGRkYXRhIjp0cnVlLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiYWRtaW4xX2NvZGUiOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiYWRtaW4yX2NvZGUiOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiYWRtaW4zX2NvZGUiOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiYWRtaW40X2NvZGUiOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiZmVhdHVyZV9jb2RlIjp7InR5cGUiOiJ0ZXh0IiwiZmllbGRzIjp7InJhdyI6eyJ0eXBlIjoia2V5d29yZCJ9fX0sImFsdGVybmF0ZW5hbWVzIjp7InR5cGUiOiJ0ZXh0IiwiZmllbGRzIjp7InJhdyI6eyJ0eXBlIjoia2V5d29yZCJ9fX0sImFzY2lpbmFtZSI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJwb3B1bGF0aW9uIjp7InR5cGUiOiJsb25nIn19fX0=" + }, + "targetResponses": [ + { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 270, + "content-type": "application/json; charset=UTF-8", + "content-length": "67", + "body": "eyJhY2tub3dsZWRnZWQiOnRydWUsInNoYXJkc19hY2tub3dsZWRnZWQiOnRydWUsImluZGV4IjoiZ2VvbmFtZXMifQ==" + } + ], + "connectionId": "0242acfffe12000b-0000000a-00000009-19ac20d3defa9804-07697cc6.0", + "numRequests": 1, + "numErrors": 0 + }, + { + "sourceRequest": { + "Request-URI": "/_cluster/health/geonames?wait_for_status=green&wait_for_no_relocating_shards=true", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46YWRtaW4=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "body": "" + }, + "sourceResponse": { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 3, + "content-type": "application/json; charset=UTF-8", + "content-length": "390", + "body": "eyJjbHVzdGVyX25hbWUiOiJkb2NrZXItY2x1c3RlciIsInN0YXR1cyI6ImdyZWVuIiwidGltZWRfb3V0IjpmYWxzZSwibnVtYmVyX29mX25vZGVzIjoxLCJudW1iZXJfb2ZfZGF0YV9ub2RlcyI6MSwiYWN0aXZlX3ByaW1hcnlfc2hhcmRzIjo1LCJhY3RpdmVfc2hhcmRzIjo1LCJyZWxvY2F0aW5nX3NoYXJkcyI6MCwiaW5pdGlhbGl6aW5nX3NoYXJkcyI6MCwidW5hc3NpZ25lZF9zaGFyZHMiOjAsImRlbGF5ZWRfdW5hc3NpZ25lZF9zaGFyZHMiOjAsIm51bWJlcl9vZl9wZW5kaW5nX3Rhc2tzIjowLCJudW1iZXJfb2ZfaW5fZmxpZ2h0X2ZldGNoIjowLCJ0YXNrX21heF93YWl0aW5nX2luX3F1ZXVlX21pbGxpcyI6MCwiYWN0aXZlX3NoYXJkc19wZXJjZW50X2FzX251bWJlciI6MTAwLjB9" + }, + "targetRequest": { + "Request-URI": "/_cluster/health/geonames?wait_for_status=green&wait_for_no_relocating_shards=true", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "body": "" + }, + "targetResponses": [ + { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 29, + "content-type": "application/json; charset=UTF-8", + "content-length": "449", + "body": "eyJjbHVzdGVyX25hbWUiOiJkb2NrZXItY2x1c3RlciIsInN0YXR1cyI6ImdyZWVuIiwidGltZWRfb3V0IjpmYWxzZSwibnVtYmVyX29mX25vZGVzIjoxLCJudW1iZXJfb2ZfZGF0YV9ub2RlcyI6MSwiZGlzY292ZXJlZF9tYXN0ZXIiOnRydWUsImRpc2NvdmVyZWRfY2x1c3Rlcl9tYW5hZ2VyIjp0cnVlLCJhY3RpdmVfcHJpbWFyeV9zaGFyZHMiOjUsImFjdGl2ZV9zaGFyZHMiOjUsInJlbG9jYXRpbmdfc2hhcmRzIjowLCJpbml0aWFsaXppbmdfc2hhcmRzIjowLCJ1bmFzc2lnbmVkX3NoYXJkcyI6MCwiZGVsYXllZF91bmFzc2lnbmVkX3NoYXJkcyI6MCwibnVtYmVyX29mX3BlbmRpbmdfdGFza3MiOjAsIm51bWJlcl9vZl9pbl9mbGlnaHRfZmV0Y2giOjAsInRhc2tfbWF4X3dhaXRpbmdfaW5fcXVldWVfbWlsbGlzIjowLCJhY3RpdmVfc2hhcmRzX3BlcmNlbnRfYXNfbnVtYmVyIjoxMDAuMH0=" + } + ], + "connectionId": "0242acfffe12000b-0000000a-0000000b-2681c86bdefa99ec-45c6b416.0", + "numRequests": 1, + "numErrors": 0 + }, + { + "sourceRequest": { + "Request-URI": "/_bulk", + "Method": "POST", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46YWRtaW4=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "3974", + "body": "eyJpbmRleCI6IHsiX2luZGV4IjogImdlb25hbWVzIn19CnsiZ2VvbmFtZWlkIjogMjk4NjA0MywgIm5hbWUiOiAiUGljIGRlIEZvbnQgQmxhbmNhIiwgImFzY2lpbmFtZSI6ICJQaWMgZGUgRm9udCBCbGFuY2EiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUGljIGRlIEZvbnQgQmxhbmNhLFBpYyBkdSBQb3J0IiwgImZlYXR1cmVfY2xhc3MiOiAiVCIsICJmZWF0dXJlX2NvZGUiOiAiUEsiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImFkbWluMV9jb2RlIjogIjAwIiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjI4NjAiLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS41MzMzNSwgNDIuNjQ5OTFdfQp7ImluZGV4IjogeyJfaW5kZXgiOiAiZ2VvbmFtZXMifX0KeyJnZW9uYW1laWQiOiAyOTkzODM4LCAibmFtZSI6ICJQaWMgZGUgTWlsLU1lbnV0IiwgImFzY2lpbmFtZSI6ICJQaWMgZGUgTWlsLU1lbnV0IiwgImFsdGVybmF0ZW5hbWVzIjogIlBpYyBkZSBNaWwtTWVudXQiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJQSyIsICJjb3VudHJ5X2NvZGUiOiAiQUQiLCAiY2MyIjogIkFELEZSIiwgImFkbWluMV9jb2RlIjogIkIzIiwgImFkbWluMl9jb2RlIjogIjA5IiwgImFkbWluM19jb2RlIjogIjA5MSIsICJhZG1pbjRfY29kZSI6ICIwOTAyNCIsICJwb3B1bGF0aW9uIjogMCwgImRlbSI6ICIyMTM4IiwgInRpbWV6b25lIjogIkV1cm9wZS9BbmRvcnJhIiwgImxvY2F0aW9uIjogWzEuNjUsIDQyLjYzMzMzXX0KeyJpbmRleCI6IHsiX2luZGV4IjogImdlb25hbWVzIn19CnsiZ2VvbmFtZWlkIjogMjk5NDcwMSwgIm5hbWUiOiAiUm9jIE3DqWzDqSIsICJhc2NpaW5hbWUiOiAiUm9jIE1lbGUiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUm9jIE1lbGUsUm9jIE1lbGVyLFJvYyBNw6lsw6kiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJNVCIsICJjb3VudHJ5X2NvZGUiOiAiQUQiLCAiY2MyIjogIkFELEZSIiwgImFkbWluMV9jb2RlIjogIjAwIiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjI4MDMiLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS43NDAyOCwgNDIuNTg3NjVdfQp7ImluZGV4IjogeyJfaW5kZXgiOiAiZ2VvbmFtZXMifX0KeyJnZW9uYW1laWQiOiAzMDA3NjgzLCAibmFtZSI6ICJQaWMgZGVzIExhbmdvdW5lbGxlcyIsICJhc2NpaW5hbWUiOiAiUGljIGRlcyBMYW5nb3VuZWxsZXMiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUGljIGRlcyBMYW5nb3VuZWxsZXMiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJQSyIsICJjb3VudHJ5X2NvZGUiOiAiQUQiLCAiY2MyIjogIkFELEZSIiwgImFkbWluMV9jb2RlIjogIjAwIiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjI2ODUiLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS40NzM2NCwgNDIuNjEyMDNdfQp7ImluZGV4IjogeyJfaW5kZXgiOiAiZ2VvbmFtZXMifX0KeyJnZW9uYW1laWQiOiAzMDE3ODMyLCAibmFtZSI6ICJQaWMgZGUgbGVzIEFiZWxsZXRlcyIsICJhc2NpaW5hbWUiOiAiUGljIGRlIGxlcyBBYmVsbGV0ZXMiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUGljIGRlIGxhIEZvbnQtTmVncmUsUGljIGRlIGxhIEZvbnQtTsOoZ3JlLFBpYyBkZSBsZXMgQWJlbGxldGVzIiwgImZlYXR1cmVfY2xhc3MiOiAiVCIsICJmZWF0dXJlX2NvZGUiOiAiUEsiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImNjMiI6ICJGUiIsICJhZG1pbjFfY29kZSI6ICJBOSIsICJhZG1pbjJfY29kZSI6ICI2NiIsICJhZG1pbjNfY29kZSI6ICI2NjMiLCAiYWRtaW40X2NvZGUiOiAiNjYxNDYiLCAicG9wdWxhdGlvbiI6IDAsICJkZW0iOiAiMjQxMSIsICJ0aW1lem9uZSI6ICJFdXJvcGUvQW5kb3JyYSIsICJsb2NhdGlvbiI6IFsxLjczMzQzLCA0Mi41MjUzNV19CnsiaW5kZXgiOiB7Il9pbmRleCI6ICJnZW9uYW1lcyJ9fQp7Imdlb25hbWVpZCI6IDMwMTc4MzMsICJuYW1lIjogIkVzdGFueSBkZSBsZXMgQWJlbGxldGVzIiwgImFzY2lpbmFtZSI6ICJFc3RhbnkgZGUgbGVzIEFiZWxsZXRlcyIsICJhbHRlcm5hdGVuYW1lcyI6ICJFc3RhbnkgZGUgbGVzIEFiZWxsZXRlcyxFdGFuZyBkZSBGb250LU5lZ3JlLMOJdGFuZyBkZSBGb250LU7DqGdyZSIsICJmZWF0dXJlX2NsYXNzIjogIkgiLCAiZmVhdHVyZV9jb2RlIjogIkxLIiwgImNvdW50cnlfY29kZSI6ICJBRCIsICJjYzIiOiAiRlIiLCAiYWRtaW4xX2NvZGUiOiAiQTkiLCAicG9wdWxhdGlvbiI6IDAsICJkZW0iOiAiMjI2MCIsICJ0aW1lem9uZSI6ICJFdXJvcGUvQW5kb3JyYSIsICJsb2NhdGlvbiI6IFsxLjczMzYyLCA0Mi41MjkxNV19CnsiaW5kZXgiOiB7Il9pbmRleCI6ICJnZW9uYW1lcyJ9fQp7Imdlb25hbWVpZCI6IDMwMjMyMDMsICJuYW1lIjogIlBvcnQgVmlldXggZGUgbGEgQ291bWUgZOKAmU9zZSIsICJhc2NpaW5hbWUiOiAiUG9ydCBWaWV1eCBkZSBsYSBDb3VtZSBkJ09zZSIsICJhbHRlcm5hdGVuYW1lcyI6ICJQb3J0IFZpZXV4IGRlIENvdW1lIGQnT3NlLFBvcnQgVmlldXggZGUgQ291bWUgZOKAmU9zZSxQb3J0IFZpZXV4IGRlIGxhIENvdW1lIGQnT3NlLFBvcnQgVmlldXggZGUgbGEgQ291bWUgZOKAmU9zZSIsICJmZWF0dXJlX2NsYXNzIjogIlQiLCAiZmVhdHVyZV9jb2RlIjogIlBBU1MiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImFkbWluMV9jb2RlIjogIjAwIiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjI2ODciLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS42MTgyMywgNDIuNjI1NjhdfQp7ImluZGV4IjogeyJfaW5kZXgiOiAiZ2VvbmFtZXMifX0KeyJnZW9uYW1laWQiOiAzMDI5MzE1LCAibmFtZSI6ICJQb3J0IGRlIGxhIENhYmFuZXR0ZSIsICJhc2NpaW5hbWUiOiAiUG9ydCBkZSBsYSBDYWJhbmV0dGUiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUG9ydCBkZSBsYSBDYWJhbmV0dGUsUG9ydGVpbGxlIGRlIGxhIENhYmFuZXR0ZSIsICJmZWF0dXJlX2NsYXNzIjogIlQiLCAiZmVhdHVyZV9jb2RlIjogIlBBU1MiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImNjMiI6ICJBRCxGUiIsICJhZG1pbjFfY29kZSI6ICJCMyIsICJhZG1pbjJfY29kZSI6ICIwOSIsICJhZG1pbjNfY29kZSI6ICIwOTEiLCAiYWRtaW40X2NvZGUiOiAiMDkxMzkiLCAicG9wdWxhdGlvbiI6IDAsICJkZW0iOiAiMjM3OSIsICJ0aW1lem9uZSI6ICJFdXJvcGUvQW5kb3JyYSIsICJsb2NhdGlvbiI6IFsxLjczMzMzLCA0Mi42XX0KeyJpbmRleCI6IHsiX2luZGV4IjogImdlb25hbWVzIn19CnsiZ2VvbmFtZWlkIjogMzAzNDk0NSwgIm5hbWUiOiAiUG9ydCBEcmV0IiwgImFzY2lpbmFtZSI6ICJQb3J0IERyZXQiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUG9ydCBEcmV0LFBvcnQgZGUgQmFyZWl0ZXMsUG9ydCBkZSBsYXMgQmFyZXl0ZXMsUG9ydCBkZXMgQmFyZXl0ZXMiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJQQVNTIiwgImNvdW50cnlfY29kZSI6ICJBRCIsICJhZG1pbjFfY29kZSI6ICIwMCIsICJwb3B1bGF0aW9uIjogMCwgImRlbSI6ICIyNjYwIiwgInRpbWV6b25lIjogIkV1cm9wZS9BbmRvcnJhIiwgImxvY2F0aW9uIjogWzEuNDU1NjIsIDQyLjYwMTcyXX0KeyJpbmRleCI6IHsiX2luZGV4IjogImdlb25hbWVzIn19CnsiZ2VvbmFtZWlkIjogMzAzODgxNCwgIm5hbWUiOiAiQ29zdGEgZGUgWHVyaXVzIiwgImFzY2lpbmFtZSI6ICJDb3N0YSBkZSBYdXJpdXMiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJTTFAiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImFkbWluMV9jb2RlIjogIjA3IiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjE4MzkiLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS40NzU2OSwgNDIuNTA2OTJdfQo=" + }, + "sourceResponse": { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 47, + "content-type": "application/json; charset=UTF-8", + "content-length": "2026", + "body": "eyJ0b29rIjozNSwiZXJyb3JzIjpmYWxzZSwiaXRlbXMiOlt7ImluZGV4Ijp7Il9pbmRleCI6Imdlb25hbWVzIiwiX3R5cGUiOiJfZG9jIiwiX2lkIjoiSTlENkM1SUJIQ0R0OTNrLW5CSXciLCJfdmVyc2lvbiI6MSwicmVzdWx0IjoiY3JlYXRlZCIsIl9zaGFyZHMiOnsidG90YWwiOjEsInN1Y2Nlc3NmdWwiOjEsImZhaWxlZCI6MH0sIl9zZXFfbm8iOjAsIl9wcmltYXJ5X3Rlcm0iOjEsInN0YXR1cyI6MjAxfX0seyJpbmRleCI6eyJfaW5kZXgiOiJnZW9uYW1lcyIsIl90eXBlIjoiX2RvYyIsIl9pZCI6IkpORDZDNUlCSENEdDkzay1uQkl4IiwiX3ZlcnNpb24iOjEsInJlc3VsdCI6ImNyZWF0ZWQiLCJfc2hhcmRzIjp7InRvdGFsIjoxLCJzdWNjZXNzZnVsIjoxLCJmYWlsZWQiOjB9LCJfc2VxX25vIjowLCJfcHJpbWFyeV90ZXJtIjoxLCJzdGF0dXMiOjIwMX19LHsiaW5kZXgiOnsiX2luZGV4IjoiZ2VvbmFtZXMiLCJfdHlwZSI6Il9kb2MiLCJfaWQiOiJKZEQ2QzVJQkhDRHQ5M2stbkJJeCIsIl92ZXJzaW9uIjoxLCJyZXN1bHQiOiJjcmVhdGVkIiwiX3NoYXJkcyI6eyJ0b3RhbCI6MSwic3VjY2Vzc2Z1bCI6MSwiZmFpbGVkIjowfSwiX3NlcV9ubyI6MSwiX3ByaW1hcnlfdGVybSI6MSwic3RhdHVzIjoyMDF9fSx7ImluZGV4Ijp7Il9pbmRleCI6Imdlb25hbWVzIiwiX3R5cGUiOiJfZG9jIiwiX2lkIjoiSnRENkM1SUJIQ0R0OTNrLW5CSXgiLCJfdmVyc2lvbiI6MSwicmVzdWx0IjoiY3JlYXRlZCIsIl9zaGFyZHMiOnsidG90YWwiOjEsInN1Y2Nlc3NmdWwiOjEsImZhaWxlZCI6MH0sIl9zZXFfbm8iOjAsIl9wcmltYXJ5X3Rlcm0iOjEsInN0YXR1cyI6MjAxfX0seyJpbmRleCI6eyJfaW5kZXgiOiJnZW9uYW1lcyIsIl90eXBlIjoiX2RvYyIsIl9pZCI6Iko5RDZDNUlCSENEdDkzay1uQkl4IiwiX3ZlcnNpb24iOjEsInJlc3VsdCI6ImNyZWF0ZWQiLCJfc2hhcmRzIjp7InRvdGFsIjoxLCJzdWNjZXNzZnVsIjoxLCJmYWlsZWQiOjB9LCJfc2VxX25vIjowLCJfcHJpbWFyeV90ZXJtIjoxLCJzdGF0dXMiOjIwMX19LHsiaW5kZXgiOnsiX2luZGV4IjoiZ2VvbmFtZXMiLCJfdHlwZSI6Il9kb2MiLCJfaWQiOiJLTkQ2QzVJQkhDRHQ5M2stbkJJeCIsIl92ZXJzaW9uIjoxLCJyZXN1bHQiOiJjcmVhdGVkIiwiX3NoYXJkcyI6eyJ0b3RhbCI6MSwic3VjY2Vzc2Z1bCI6MSwiZmFpbGVkIjowfSwiX3NlcV9ubyI6MiwiX3ByaW1hcnlfdGVybSI6MSwic3RhdHVzIjoyMDF9fSx7ImluZGV4Ijp7Il9pbmRleCI6Imdlb25hbWVzIiwiX3R5cGUiOiJfZG9jIiwiX2lkIjoiS2RENkM1SUJIQ0R0OTNrLW5CSXgiLCJfdmVyc2lvbiI6MSwicmVzdWx0IjoiY3JlYXRlZCIsIl9zaGFyZHMiOnsidG90YWwiOjEsInN1Y2Nlc3NmdWwiOjEsImZhaWxlZCI6MH0sIl9zZXFfbm8iOjEsIl9wcmltYXJ5X3Rlcm0iOjEsInN0YXR1cyI6MjAxfX0seyJpbmRleCI6eyJfaW5kZXgiOiJnZW9uYW1lcyIsIl90eXBlIjoiX2RvYyIsIl9pZCI6Ikt0RDZDNUlCSENEdDkzay1uQkl4IiwiX3ZlcnNpb24iOjEsInJlc3VsdCI6ImNyZWF0ZWQiLCJfc2hhcmRzIjp7InRvdGFsIjoxLCJzdWNjZXNzZnVsIjoxLCJmYWlsZWQiOjB9LCJfc2VxX25vIjozLCJfcHJpbWFyeV90ZXJtIjoxLCJzdGF0dXMiOjIwMX19LHsiaW5kZXgiOnsiX2luZGV4IjoiZ2VvbmFtZXMiLCJfdHlwZSI6Il9kb2MiLCJfaWQiOiJLOUQ2QzVJQkhDRHQ5M2stbkJJeCIsIl92ZXJzaW9uIjoxLCJyZXN1bHQiOiJjcmVhdGVkIiwiX3NoYXJkcyI6eyJ0b3RhbCI6MSwic3VjY2Vzc2Z1bCI6MSwiZmFpbGVkIjowfSwiX3NlcV9ubyI6MSwiX3ByaW1hcnlfdGVybSI6MSwic3RhdHVzIjoyMDF9fSx7ImluZGV4Ijp7Il9pbmRleCI6Imdlb25hbWVzIiwiX3R5cGUiOiJfZG9jIiwiX2lkIjoiTE5ENkM1SUJIQ0R0OTNrLW5CSXgiLCJfdmVyc2lvbiI6MSwicmVzdWx0IjoiY3JlYXRlZCIsIl9zaGFyZHMiOnsidG90YWwiOjEsInN1Y2Nlc3NmdWwiOjEsImZhaWxlZCI6MH0sIl9zZXFfbm8iOjAsIl9wcmltYXJ5X3Rlcm0iOjEsInN0YXR1cyI6MjAxfX1dfQ==" + }, + "targetRequest": { + "Request-URI": "/_bulk", + "Method": "POST", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "3974", + "body": "eyJpbmRleCI6IHsiX2luZGV4IjogImdlb25hbWVzIn19CnsiZ2VvbmFtZWlkIjogMjk4NjA0MywgIm5hbWUiOiAiUGljIGRlIEZvbnQgQmxhbmNhIiwgImFzY2lpbmFtZSI6ICJQaWMgZGUgRm9udCBCbGFuY2EiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUGljIGRlIEZvbnQgQmxhbmNhLFBpYyBkdSBQb3J0IiwgImZlYXR1cmVfY2xhc3MiOiAiVCIsICJmZWF0dXJlX2NvZGUiOiAiUEsiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImFkbWluMV9jb2RlIjogIjAwIiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjI4NjAiLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS41MzMzNSwgNDIuNjQ5OTFdfQp7ImluZGV4IjogeyJfaW5kZXgiOiAiZ2VvbmFtZXMifX0KeyJnZW9uYW1laWQiOiAyOTkzODM4LCAibmFtZSI6ICJQaWMgZGUgTWlsLU1lbnV0IiwgImFzY2lpbmFtZSI6ICJQaWMgZGUgTWlsLU1lbnV0IiwgImFsdGVybmF0ZW5hbWVzIjogIlBpYyBkZSBNaWwtTWVudXQiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJQSyIsICJjb3VudHJ5X2NvZGUiOiAiQUQiLCAiY2MyIjogIkFELEZSIiwgImFkbWluMV9jb2RlIjogIkIzIiwgImFkbWluMl9jb2RlIjogIjA5IiwgImFkbWluM19jb2RlIjogIjA5MSIsICJhZG1pbjRfY29kZSI6ICIwOTAyNCIsICJwb3B1bGF0aW9uIjogMCwgImRlbSI6ICIyMTM4IiwgInRpbWV6b25lIjogIkV1cm9wZS9BbmRvcnJhIiwgImxvY2F0aW9uIjogWzEuNjUsIDQyLjYzMzMzXX0KeyJpbmRleCI6IHsiX2luZGV4IjogImdlb25hbWVzIn19CnsiZ2VvbmFtZWlkIjogMjk5NDcwMSwgIm5hbWUiOiAiUm9jIE3DqWzDqSIsICJhc2NpaW5hbWUiOiAiUm9jIE1lbGUiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUm9jIE1lbGUsUm9jIE1lbGVyLFJvYyBNw6lsw6kiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJNVCIsICJjb3VudHJ5X2NvZGUiOiAiQUQiLCAiY2MyIjogIkFELEZSIiwgImFkbWluMV9jb2RlIjogIjAwIiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjI4MDMiLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS43NDAyOCwgNDIuNTg3NjVdfQp7ImluZGV4IjogeyJfaW5kZXgiOiAiZ2VvbmFtZXMifX0KeyJnZW9uYW1laWQiOiAzMDA3NjgzLCAibmFtZSI6ICJQaWMgZGVzIExhbmdvdW5lbGxlcyIsICJhc2NpaW5hbWUiOiAiUGljIGRlcyBMYW5nb3VuZWxsZXMiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUGljIGRlcyBMYW5nb3VuZWxsZXMiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJQSyIsICJjb3VudHJ5X2NvZGUiOiAiQUQiLCAiY2MyIjogIkFELEZSIiwgImFkbWluMV9jb2RlIjogIjAwIiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjI2ODUiLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS40NzM2NCwgNDIuNjEyMDNdfQp7ImluZGV4IjogeyJfaW5kZXgiOiAiZ2VvbmFtZXMifX0KeyJnZW9uYW1laWQiOiAzMDE3ODMyLCAibmFtZSI6ICJQaWMgZGUgbGVzIEFiZWxsZXRlcyIsICJhc2NpaW5hbWUiOiAiUGljIGRlIGxlcyBBYmVsbGV0ZXMiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUGljIGRlIGxhIEZvbnQtTmVncmUsUGljIGRlIGxhIEZvbnQtTsOoZ3JlLFBpYyBkZSBsZXMgQWJlbGxldGVzIiwgImZlYXR1cmVfY2xhc3MiOiAiVCIsICJmZWF0dXJlX2NvZGUiOiAiUEsiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImNjMiI6ICJGUiIsICJhZG1pbjFfY29kZSI6ICJBOSIsICJhZG1pbjJfY29kZSI6ICI2NiIsICJhZG1pbjNfY29kZSI6ICI2NjMiLCAiYWRtaW40X2NvZGUiOiAiNjYxNDYiLCAicG9wdWxhdGlvbiI6IDAsICJkZW0iOiAiMjQxMSIsICJ0aW1lem9uZSI6ICJFdXJvcGUvQW5kb3JyYSIsICJsb2NhdGlvbiI6IFsxLjczMzQzLCA0Mi41MjUzNV19CnsiaW5kZXgiOiB7Il9pbmRleCI6ICJnZW9uYW1lcyJ9fQp7Imdlb25hbWVpZCI6IDMwMTc4MzMsICJuYW1lIjogIkVzdGFueSBkZSBsZXMgQWJlbGxldGVzIiwgImFzY2lpbmFtZSI6ICJFc3RhbnkgZGUgbGVzIEFiZWxsZXRlcyIsICJhbHRlcm5hdGVuYW1lcyI6ICJFc3RhbnkgZGUgbGVzIEFiZWxsZXRlcyxFdGFuZyBkZSBGb250LU5lZ3JlLMOJdGFuZyBkZSBGb250LU7DqGdyZSIsICJmZWF0dXJlX2NsYXNzIjogIkgiLCAiZmVhdHVyZV9jb2RlIjogIkxLIiwgImNvdW50cnlfY29kZSI6ICJBRCIsICJjYzIiOiAiRlIiLCAiYWRtaW4xX2NvZGUiOiAiQTkiLCAicG9wdWxhdGlvbiI6IDAsICJkZW0iOiAiMjI2MCIsICJ0aW1lem9uZSI6ICJFdXJvcGUvQW5kb3JyYSIsICJsb2NhdGlvbiI6IFsxLjczMzYyLCA0Mi41MjkxNV19CnsiaW5kZXgiOiB7Il9pbmRleCI6ICJnZW9uYW1lcyJ9fQp7Imdlb25hbWVpZCI6IDMwMjMyMDMsICJuYW1lIjogIlBvcnQgVmlldXggZGUgbGEgQ291bWUgZOKAmU9zZSIsICJhc2NpaW5hbWUiOiAiUG9ydCBWaWV1eCBkZSBsYSBDb3VtZSBkJ09zZSIsICJhbHRlcm5hdGVuYW1lcyI6ICJQb3J0IFZpZXV4IGRlIENvdW1lIGQnT3NlLFBvcnQgVmlldXggZGUgQ291bWUgZOKAmU9zZSxQb3J0IFZpZXV4IGRlIGxhIENvdW1lIGQnT3NlLFBvcnQgVmlldXggZGUgbGEgQ291bWUgZOKAmU9zZSIsICJmZWF0dXJlX2NsYXNzIjogIlQiLCAiZmVhdHVyZV9jb2RlIjogIlBBU1MiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImFkbWluMV9jb2RlIjogIjAwIiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjI2ODciLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS42MTgyMywgNDIuNjI1NjhdfQp7ImluZGV4IjogeyJfaW5kZXgiOiAiZ2VvbmFtZXMifX0KeyJnZW9uYW1laWQiOiAzMDI5MzE1LCAibmFtZSI6ICJQb3J0IGRlIGxhIENhYmFuZXR0ZSIsICJhc2NpaW5hbWUiOiAiUG9ydCBkZSBsYSBDYWJhbmV0dGUiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUG9ydCBkZSBsYSBDYWJhbmV0dGUsUG9ydGVpbGxlIGRlIGxhIENhYmFuZXR0ZSIsICJmZWF0dXJlX2NsYXNzIjogIlQiLCAiZmVhdHVyZV9jb2RlIjogIlBBU1MiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImNjMiI6ICJBRCxGUiIsICJhZG1pbjFfY29kZSI6ICJCMyIsICJhZG1pbjJfY29kZSI6ICIwOSIsICJhZG1pbjNfY29kZSI6ICIwOTEiLCAiYWRtaW40X2NvZGUiOiAiMDkxMzkiLCAicG9wdWxhdGlvbiI6IDAsICJkZW0iOiAiMjM3OSIsICJ0aW1lem9uZSI6ICJFdXJvcGUvQW5kb3JyYSIsICJsb2NhdGlvbiI6IFsxLjczMzMzLCA0Mi42XX0KeyJpbmRleCI6IHsiX2luZGV4IjogImdlb25hbWVzIn19CnsiZ2VvbmFtZWlkIjogMzAzNDk0NSwgIm5hbWUiOiAiUG9ydCBEcmV0IiwgImFzY2lpbmFtZSI6ICJQb3J0IERyZXQiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUG9ydCBEcmV0LFBvcnQgZGUgQmFyZWl0ZXMsUG9ydCBkZSBsYXMgQmFyZXl0ZXMsUG9ydCBkZXMgQmFyZXl0ZXMiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJQQVNTIiwgImNvdW50cnlfY29kZSI6ICJBRCIsICJhZG1pbjFfY29kZSI6ICIwMCIsICJwb3B1bGF0aW9uIjogMCwgImRlbSI6ICIyNjYwIiwgInRpbWV6b25lIjogIkV1cm9wZS9BbmRvcnJhIiwgImxvY2F0aW9uIjogWzEuNDU1NjIsIDQyLjYwMTcyXX0KeyJpbmRleCI6IHsiX2luZGV4IjogImdlb25hbWVzIn19CnsiZ2VvbmFtZWlkIjogMzAzODgxNCwgIm5hbWUiOiAiQ29zdGEgZGUgWHVyaXVzIiwgImFzY2lpbmFtZSI6ICJDb3N0YSBkZSBYdXJpdXMiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJTTFAiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImFkbWluMV9jb2RlIjogIjA3IiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjE4MzkiLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS40NzU2OSwgNDIuNTA2OTJdfQo=" + }, + "targetResponses": [ + { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 49, + "content-type": "application/json; charset=UTF-8", + "content-length": "1876", + "body": "eyJ0b29rIjoyNSwiZXJyb3JzIjpmYWxzZSwiaXRlbXMiOlt7ImluZGV4Ijp7Il9pbmRleCI6Imdlb25hbWVzIiwiX2lkIjoiMFlfNkM1SUIzdGszODYtVG5NOUYiLCJfdmVyc2lvbiI6MSwicmVzdWx0IjoiY3JlYXRlZCIsIl9zaGFyZHMiOnsidG90YWwiOjEsInN1Y2Nlc3NmdWwiOjEsImZhaWxlZCI6MH0sIl9zZXFfbm8iOjAsIl9wcmltYXJ5X3Rlcm0iOjEsInN0YXR1cyI6MjAxfX0seyJpbmRleCI6eyJfaW5kZXgiOiJnZW9uYW1lcyIsIl9pZCI6IjBvXzZDNUlCM3RrMzg2LVRuTTlGIiwiX3ZlcnNpb24iOjEsInJlc3VsdCI6ImNyZWF0ZWQiLCJfc2hhcmRzIjp7InRvdGFsIjoxLCJzdWNjZXNzZnVsIjoxLCJmYWlsZWQiOjB9LCJfc2VxX25vIjowLCJfcHJpbWFyeV90ZXJtIjoxLCJzdGF0dXMiOjIwMX19LHsiaW5kZXgiOnsiX2luZGV4IjoiZ2VvbmFtZXMiLCJfaWQiOiIwNF82QzVJQjN0azM4Ni1Ubk05RiIsIl92ZXJzaW9uIjoxLCJyZXN1bHQiOiJjcmVhdGVkIiwiX3NoYXJkcyI6eyJ0b3RhbCI6MSwic3VjY2Vzc2Z1bCI6MSwiZmFpbGVkIjowfSwiX3NlcV9ubyI6MSwiX3ByaW1hcnlfdGVybSI6MSwic3RhdHVzIjoyMDF9fSx7ImluZGV4Ijp7Il9pbmRleCI6Imdlb25hbWVzIiwiX2lkIjoiMUlfNkM1SUIzdGszODYtVG5NOUYiLCJfdmVyc2lvbiI6MSwicmVzdWx0IjoiY3JlYXRlZCIsIl9zaGFyZHMiOnsidG90YWwiOjEsInN1Y2Nlc3NmdWwiOjEsImZhaWxlZCI6MH0sIl9zZXFfbm8iOjIsIl9wcmltYXJ5X3Rlcm0iOjEsInN0YXR1cyI6MjAxfX0seyJpbmRleCI6eyJfaW5kZXgiOiJnZW9uYW1lcyIsIl9pZCI6IjFZXzZDNUlCM3RrMzg2LVRuTTlGIiwiX3ZlcnNpb24iOjEsInJlc3VsdCI6ImNyZWF0ZWQiLCJfc2hhcmRzIjp7InRvdGFsIjoxLCJzdWNjZXNzZnVsIjoxLCJmYWlsZWQiOjB9LCJfc2VxX25vIjowLCJfcHJpbWFyeV90ZXJtIjoxLCJzdGF0dXMiOjIwMX19LHsiaW5kZXgiOnsiX2luZGV4IjoiZ2VvbmFtZXMiLCJfaWQiOiIxb182QzVJQjN0azM4Ni1Ubk05RiIsIl92ZXJzaW9uIjoxLCJyZXN1bHQiOiJjcmVhdGVkIiwiX3NoYXJkcyI6eyJ0b3RhbCI6MSwic3VjY2Vzc2Z1bCI6MSwiZmFpbGVkIjowfSwiX3NlcV9ubyI6MCwiX3ByaW1hcnlfdGVybSI6MSwic3RhdHVzIjoyMDF9fSx7ImluZGV4Ijp7Il9pbmRleCI6Imdlb25hbWVzIiwiX2lkIjoiMTRfNkM1SUIzdGszODYtVG5NOUYiLCJfdmVyc2lvbiI6MSwicmVzdWx0IjoiY3JlYXRlZCIsIl9zaGFyZHMiOnsidG90YWwiOjEsInN1Y2Nlc3NmdWwiOjEsImZhaWxlZCI6MH0sIl9zZXFfbm8iOjEsIl9wcmltYXJ5X3Rlcm0iOjEsInN0YXR1cyI6MjAxfX0seyJpbmRleCI6eyJfaW5kZXgiOiJnZW9uYW1lcyIsIl9pZCI6IjJJXzZDNUlCM3RrMzg2LVRuTTlGIiwiX3ZlcnNpb24iOjEsInJlc3VsdCI6ImNyZWF0ZWQiLCJfc2hhcmRzIjp7InRvdGFsIjoxLCJzdWNjZXNzZnVsIjoxLCJmYWlsZWQiOjB9LCJfc2VxX25vIjoyLCJfcHJpbWFyeV90ZXJtIjoxLCJzdGF0dXMiOjIwMX19LHsiaW5kZXgiOnsiX2luZGV4IjoiZ2VvbmFtZXMiLCJfaWQiOiIyWV82QzVJQjN0azM4Ni1Ubk05RiIsIl92ZXJzaW9uIjoxLCJyZXN1bHQiOiJjcmVhdGVkIiwiX3NoYXJkcyI6eyJ0b3RhbCI6MSwic3VjY2Vzc2Z1bCI6MSwiZmFpbGVkIjowfSwiX3NlcV9ubyI6MywiX3ByaW1hcnlfdGVybSI6MSwic3RhdHVzIjoyMDF9fSx7ImluZGV4Ijp7Il9pbmRleCI6Imdlb25hbWVzIiwiX2lkIjoiMm9fNkM1SUIzdGszODYtVG5NOUYiLCJfdmVyc2lvbiI6MSwicmVzdWx0IjoiY3JlYXRlZCIsIl9zaGFyZHMiOnsidG90YWwiOjEsInN1Y2Nlc3NmdWwiOjEsImZhaWxlZCI6MH0sIl9zZXFfbm8iOjAsIl9wcmltYXJ5X3Rlcm0iOjEsInN0YXR1cyI6MjAxfX1dfQ==" + } + ], + "connectionId": "0242acfffe12000b-0000000a-0000000d-380cb36fdefa9c00-e2a1d15a.0", + "numRequests": 1, + "numErrors": 0 + } +] diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/multiple_tuples_parsed.json b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/multiple_tuples_parsed.json new file mode 100644 index 000000000..464afd0cb --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/multiple_tuples_parsed.json @@ -0,0 +1,1018 @@ +[ + { + "sourceRequest": { + "Request-URI": "/", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "User-Agent": "python-requests/2.32.3", + "Accept-Encoding": "gzip, deflate, zstd", + "Accept": "*/*", + "Connection": "keep-alive", + "Authorization": "Basic YWRtaW46YWRtaW4=", + "body": "" + }, + "sourceResponse": { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 374, + "content-type": "application/json; charset=UTF-8", + "content-length": "538", + "body": { + "name": "3cc068ad54eb", + "cluster_name": "docker-cluster", + "cluster_uuid": "PIAawDwSQKSRLtoaXC-oWg", + "version": { + "number": "7.10.2", + "build_flavor": "oss", + "build_type": "docker", + "build_hash": "747e1cc71def077253878a59143c1f785afa92b9", + "build_date": "2021-01-13T00:42:12.435326Z", + "build_snapshot": false, + "lucene_version": "8.7.0", + "minimum_wire_compatibility_version": "6.8.0", + "minimum_index_compatibility_version": "6.0.0-beta1" + }, + "tagline": "You Know, for Search" + } + }, + "targetRequest": { + "Request-URI": "/", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "User-Agent": "python-requests/2.32.3", + "Accept-Encoding": "gzip, deflate, zstd", + "Accept": "*/*", + "Connection": "keep-alive", + "Authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "body": "" + }, + "targetResponses": [ + { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 341, + "content-type": "application/json; charset=UTF-8", + "content-length": "568", + "body": { + "name": "e6b3a4370763", + "cluster_name": "docker-cluster", + "cluster_uuid": "88zlik7ORFuFT78zduradA", + "version": { + "distribution": "opensearch", + "number": "2.15.0", + "build_type": "tar", + "build_hash": "61dbcd0795c9bfe9b81e5762175414bc38bbcadf", + "build_date": "2024-06-20T03:27:32.562036890Z", + "build_snapshot": false, + "lucene_version": "9.10.0", + "minimum_wire_compatibility_version": "7.10.0", + "minimum_index_compatibility_version": "7.0.0" + }, + "tagline": "The OpenSearch Project: https://opensearch.org/" + } + } + ], + "connectionId": "0242acfffe12000b-0000000a-00000003-4c7bb8f4aefa3189-c0544dbf.0", + "numRequests": 1, + "numErrors": 0 + }, + { + "sourceRequest": { + "Request-URI": "/geonames", + "Method": "PUT", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46YWRtaW4=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "1214", + "body": { + "settings": { + "index.number_of_shards": 5, + "index.number_of_replicas": 0, + "index.store.type": "fs", + "index.queries.cache.enabled": false, + "index.requests.cache.enable": false + }, + "mappings": { + "dynamic": "strict", + "_source": { "enabled": true }, + "properties": { + "elevation": { "type": "integer" }, + "name": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "geonameid": { "type": "long" }, + "feature_class": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "location": { "type": "geo_point" }, + "cc2": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "timezone": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "dem": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "country_code": { + "type": "text", + "fielddata": true, + "fields": { "raw": { "type": "keyword" } } + }, + "admin1_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "admin2_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "admin3_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "admin4_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "feature_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "alternatenames": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "asciiname": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "population": { "type": "long" } + } + } + } + }, + "sourceResponse": { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 232, + "content-type": "application/json; charset=UTF-8", + "content-length": "67", + "body": { + "acknowledged": true, + "shards_acknowledged": true, + "index": "geonames" + } + }, + "targetRequest": { + "Request-URI": "/geonames", + "Method": "PUT", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "1214", + "body": { + "settings": { + "index.number_of_shards": 5, + "index.number_of_replicas": 0, + "index.store.type": "fs", + "index.queries.cache.enabled": false, + "index.requests.cache.enable": false + }, + "mappings": { + "dynamic": "strict", + "_source": { "enabled": true }, + "properties": { + "elevation": { "type": "integer" }, + "name": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "geonameid": { "type": "long" }, + "feature_class": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "location": { "type": "geo_point" }, + "cc2": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "timezone": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "dem": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "country_code": { + "type": "text", + "fielddata": true, + "fields": { "raw": { "type": "keyword" } } + }, + "admin1_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "admin2_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "admin3_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "admin4_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "feature_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "alternatenames": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "asciiname": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "population": { "type": "long" } + } + } + } + }, + "targetResponses": [ + { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 270, + "content-type": "application/json; charset=UTF-8", + "content-length": "67", + "body": { + "acknowledged": true, + "shards_acknowledged": true, + "index": "geonames" + } + } + ], + "connectionId": "0242acfffe12000b-0000000a-00000009-19ac20d3defa9804-07697cc6.0", + "numRequests": 1, + "numErrors": 0 + }, + { + "sourceRequest": { + "Request-URI": "/_cluster/health/geonames?wait_for_status=green&wait_for_no_relocating_shards=true", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46YWRtaW4=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "body": "" + }, + "sourceResponse": { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 3, + "content-type": "application/json; charset=UTF-8", + "content-length": "390", + "body": { + "cluster_name": "docker-cluster", + "status": "green", + "timed_out": false, + "number_of_nodes": 1, + "number_of_data_nodes": 1, + "active_primary_shards": 5, + "active_shards": 5, + "relocating_shards": 0, + "initializing_shards": 0, + "unassigned_shards": 0, + "delayed_unassigned_shards": 0, + "number_of_pending_tasks": 0, + "number_of_in_flight_fetch": 0, + "task_max_waiting_in_queue_millis": 0, + "active_shards_percent_as_number": 100.0 + } + }, + "targetRequest": { + "Request-URI": "/_cluster/health/geonames?wait_for_status=green&wait_for_no_relocating_shards=true", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "body": "" + }, + "targetResponses": [ + { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 29, + "content-type": "application/json; charset=UTF-8", + "content-length": "449", + "body": { + "cluster_name": "docker-cluster", + "status": "green", + "timed_out": false, + "number_of_nodes": 1, + "number_of_data_nodes": 1, + "discovered_master": true, + "discovered_cluster_manager": true, + "active_primary_shards": 5, + "active_shards": 5, + "relocating_shards": 0, + "initializing_shards": 0, + "unassigned_shards": 0, + "delayed_unassigned_shards": 0, + "number_of_pending_tasks": 0, + "number_of_in_flight_fetch": 0, + "task_max_waiting_in_queue_millis": 0, + "active_shards_percent_as_number": 100.0 + } + } + ], + "connectionId": "0242acfffe12000b-0000000a-0000000b-2681c86bdefa99ec-45c6b416.0", + "numRequests": 1, + "numErrors": 0 + }, + { + "sourceRequest": { + "Request-URI": "/_bulk", + "Method": "POST", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46YWRtaW4=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "3974", + "body": [ + { "index": { "_index": "geonames" } }, + { + "geonameid": 2986043, + "name": "Pic de Font Blanca", + "asciiname": "Pic de Font Blanca", + "alternatenames": "Pic de Font Blanca,Pic du Port", + "feature_class": "T", + "feature_code": "PK", + "country_code": "AD", + "admin1_code": "00", + "population": 0, + "dem": "2860", + "timezone": "Europe/Andorra", + "location": [1.53335, 42.64991] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 2993838, + "name": "Pic de Mil-Menut", + "asciiname": "Pic de Mil-Menut", + "alternatenames": "Pic de Mil-Menut", + "feature_class": "T", + "feature_code": "PK", + "country_code": "AD", + "cc2": "AD,FR", + "admin1_code": "B3", + "admin2_code": "09", + "admin3_code": "091", + "admin4_code": "09024", + "population": 0, + "dem": "2138", + "timezone": "Europe/Andorra", + "location": [1.65, 42.63333] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 2994701, + "name": "Roc M\u00e9l\u00e9", + "asciiname": "Roc Mele", + "alternatenames": "Roc Mele,Roc Meler,Roc M\u00e9l\u00e9", + "feature_class": "T", + "feature_code": "MT", + "country_code": "AD", + "cc2": "AD,FR", + "admin1_code": "00", + "population": 0, + "dem": "2803", + "timezone": "Europe/Andorra", + "location": [1.74028, 42.58765] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3007683, + "name": "Pic des Langounelles", + "asciiname": "Pic des Langounelles", + "alternatenames": "Pic des Langounelles", + "feature_class": "T", + "feature_code": "PK", + "country_code": "AD", + "cc2": "AD,FR", + "admin1_code": "00", + "population": 0, + "dem": "2685", + "timezone": "Europe/Andorra", + "location": [1.47364, 42.61203] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3017832, + "name": "Pic de les Abelletes", + "asciiname": "Pic de les Abelletes", + "alternatenames": "Pic de la Font-Negre,Pic de la Font-N\u00e8gre,Pic de les Abelletes", + "feature_class": "T", + "feature_code": "PK", + "country_code": "AD", + "cc2": "FR", + "admin1_code": "A9", + "admin2_code": "66", + "admin3_code": "663", + "admin4_code": "66146", + "population": 0, + "dem": "2411", + "timezone": "Europe/Andorra", + "location": [1.73343, 42.52535] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3017833, + "name": "Estany de les Abelletes", + "asciiname": "Estany de les Abelletes", + "alternatenames": "Estany de les Abelletes,Etang de Font-Negre,\u00c9tang de Font-N\u00e8gre", + "feature_class": "H", + "feature_code": "LK", + "country_code": "AD", + "cc2": "FR", + "admin1_code": "A9", + "population": 0, + "dem": "2260", + "timezone": "Europe/Andorra", + "location": [1.73362, 42.52915] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3023203, + "name": "Port Vieux de la Coume d\u2019Ose", + "asciiname": "Port Vieux de la Coume d'Ose", + "alternatenames": "Port Vieux de Coume d'Ose,Port Vieux de Coume d\u2019Ose,Port Vieux de la Coume d'Ose,Port Vieux de la Coume d\u2019Ose", + "feature_class": "T", + "feature_code": "PASS", + "country_code": "AD", + "admin1_code": "00", + "population": 0, + "dem": "2687", + "timezone": "Europe/Andorra", + "location": [1.61823, 42.62568] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3029315, + "name": "Port de la Cabanette", + "asciiname": "Port de la Cabanette", + "alternatenames": "Port de la Cabanette,Porteille de la Cabanette", + "feature_class": "T", + "feature_code": "PASS", + "country_code": "AD", + "cc2": "AD,FR", + "admin1_code": "B3", + "admin2_code": "09", + "admin3_code": "091", + "admin4_code": "09139", + "population": 0, + "dem": "2379", + "timezone": "Europe/Andorra", + "location": [1.73333, 42.6] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3034945, + "name": "Port Dret", + "asciiname": "Port Dret", + "alternatenames": "Port Dret,Port de Bareites,Port de las Bareytes,Port des Bareytes", + "feature_class": "T", + "feature_code": "PASS", + "country_code": "AD", + "admin1_code": "00", + "population": 0, + "dem": "2660", + "timezone": "Europe/Andorra", + "location": [1.45562, 42.60172] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3038814, + "name": "Costa de Xurius", + "asciiname": "Costa de Xurius", + "feature_class": "T", + "feature_code": "SLP", + "country_code": "AD", + "admin1_code": "07", + "population": 0, + "dem": "1839", + "timezone": "Europe/Andorra", + "location": [1.47569, 42.50692] + } + ] + }, + "sourceResponse": { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 47, + "content-type": "application/json; charset=UTF-8", + "content-length": "2026", + "body": [ + { + "took": 35, + "errors": false, + "items": [ + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "I9D6C5IBHCDt93k-nBIw", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "JND6C5IBHCDt93k-nBIx", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "JdD6C5IBHCDt93k-nBIx", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 1, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "JtD6C5IBHCDt93k-nBIx", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "J9D6C5IBHCDt93k-nBIx", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "KND6C5IBHCDt93k-nBIx", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 2, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "KdD6C5IBHCDt93k-nBIx", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 1, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "KtD6C5IBHCDt93k-nBIx", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 3, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "K9D6C5IBHCDt93k-nBIx", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 1, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "LND6C5IBHCDt93k-nBIx", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + } + ] + } + ] + }, + "targetRequest": { + "Request-URI": "/_bulk", + "Method": "POST", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "3974", + "body": [ + { "index": { "_index": "geonames" } }, + { + "geonameid": 2986043, + "name": "Pic de Font Blanca", + "asciiname": "Pic de Font Blanca", + "alternatenames": "Pic de Font Blanca,Pic du Port", + "feature_class": "T", + "feature_code": "PK", + "country_code": "AD", + "admin1_code": "00", + "population": 0, + "dem": "2860", + "timezone": "Europe/Andorra", + "location": [1.53335, 42.64991] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 2993838, + "name": "Pic de Mil-Menut", + "asciiname": "Pic de Mil-Menut", + "alternatenames": "Pic de Mil-Menut", + "feature_class": "T", + "feature_code": "PK", + "country_code": "AD", + "cc2": "AD,FR", + "admin1_code": "B3", + "admin2_code": "09", + "admin3_code": "091", + "admin4_code": "09024", + "population": 0, + "dem": "2138", + "timezone": "Europe/Andorra", + "location": [1.65, 42.63333] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 2994701, + "name": "Roc M\u00e9l\u00e9", + "asciiname": "Roc Mele", + "alternatenames": "Roc Mele,Roc Meler,Roc M\u00e9l\u00e9", + "feature_class": "T", + "feature_code": "MT", + "country_code": "AD", + "cc2": "AD,FR", + "admin1_code": "00", + "population": 0, + "dem": "2803", + "timezone": "Europe/Andorra", + "location": [1.74028, 42.58765] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3007683, + "name": "Pic des Langounelles", + "asciiname": "Pic des Langounelles", + "alternatenames": "Pic des Langounelles", + "feature_class": "T", + "feature_code": "PK", + "country_code": "AD", + "cc2": "AD,FR", + "admin1_code": "00", + "population": 0, + "dem": "2685", + "timezone": "Europe/Andorra", + "location": [1.47364, 42.61203] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3017832, + "name": "Pic de les Abelletes", + "asciiname": "Pic de les Abelletes", + "alternatenames": "Pic de la Font-Negre,Pic de la Font-N\u00e8gre,Pic de les Abelletes", + "feature_class": "T", + "feature_code": "PK", + "country_code": "AD", + "cc2": "FR", + "admin1_code": "A9", + "admin2_code": "66", + "admin3_code": "663", + "admin4_code": "66146", + "population": 0, + "dem": "2411", + "timezone": "Europe/Andorra", + "location": [1.73343, 42.52535] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3017833, + "name": "Estany de les Abelletes", + "asciiname": "Estany de les Abelletes", + "alternatenames": "Estany de les Abelletes,Etang de Font-Negre,\u00c9tang de Font-N\u00e8gre", + "feature_class": "H", + "feature_code": "LK", + "country_code": "AD", + "cc2": "FR", + "admin1_code": "A9", + "population": 0, + "dem": "2260", + "timezone": "Europe/Andorra", + "location": [1.73362, 42.52915] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3023203, + "name": "Port Vieux de la Coume d\u2019Ose", + "asciiname": "Port Vieux de la Coume d'Ose", + "alternatenames": "Port Vieux de Coume d'Ose,Port Vieux de Coume d\u2019Ose,Port Vieux de la Coume d'Ose,Port Vieux de la Coume d\u2019Ose", + "feature_class": "T", + "feature_code": "PASS", + "country_code": "AD", + "admin1_code": "00", + "population": 0, + "dem": "2687", + "timezone": "Europe/Andorra", + "location": [1.61823, 42.62568] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3029315, + "name": "Port de la Cabanette", + "asciiname": "Port de la Cabanette", + "alternatenames": "Port de la Cabanette,Porteille de la Cabanette", + "feature_class": "T", + "feature_code": "PASS", + "country_code": "AD", + "cc2": "AD,FR", + "admin1_code": "B3", + "admin2_code": "09", + "admin3_code": "091", + "admin4_code": "09139", + "population": 0, + "dem": "2379", + "timezone": "Europe/Andorra", + "location": [1.73333, 42.6] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3034945, + "name": "Port Dret", + "asciiname": "Port Dret", + "alternatenames": "Port Dret,Port de Bareites,Port de las Bareytes,Port des Bareytes", + "feature_class": "T", + "feature_code": "PASS", + "country_code": "AD", + "admin1_code": "00", + "population": 0, + "dem": "2660", + "timezone": "Europe/Andorra", + "location": [1.45562, 42.60172] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3038814, + "name": "Costa de Xurius", + "asciiname": "Costa de Xurius", + "feature_class": "T", + "feature_code": "SLP", + "country_code": "AD", + "admin1_code": "07", + "population": 0, + "dem": "1839", + "timezone": "Europe/Andorra", + "location": [1.47569, 42.50692] + } + ] + }, + "targetResponses": [ + { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 49, + "content-type": "application/json; charset=UTF-8", + "content-length": "1876", + "body": [ + { + "took": 25, + "errors": false, + "items": [ + { + "index": { + "_index": "geonames", + "_id": "0Y_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_id": "0o_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_id": "04_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 1, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_id": "1I_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 2, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_id": "1Y_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_id": "1o_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_id": "14_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 1, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_id": "2I_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 2, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_id": "2Y_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 3, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_id": "2o_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + } + ] + } + ] + } + ], + "connectionId": "0242acfffe12000b-0000000a-0000000d-380cb36fdefa9c00-e2a1d15a.0", + "numRequests": 1, + "numErrors": 0 + } +] diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/valid_tuple.json b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/valid_tuple.json new file mode 100644 index 000000000..8d7af4d13 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/valid_tuple.json @@ -0,0 +1,49 @@ +{ + "sourceRequest": { + "Request-URI": "/_cat/indices?v", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "Authorization": "Basic YWRtaW46YWRtaW4=", + "User-Agent": "curl/8.5.0", + "Accept": "*/*", + "body": "" + }, + "sourceResponse": { + "HTTP-Version": { + "keepAliveDefault": true + }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 59, + "content-type": "text/plain; charset=UTF-8", + "content-length": "214", + "body": "aGVhbHRoIHN0YXR1cyBpbmRleCAgICAgICB1dWlkICAgICAgICAgICAgICAgICAgIHByaSByZXAgZG9jcy5jb3VudCBkb2NzLmRlbGV0ZWQgc3RvcmUuc2l6ZSBwcmkuc3RvcmUuc2l6ZQpncmVlbiAgb3BlbiAgIHNlYXJjaGd1YXJkIHlKS1hQTUh0VFJPTklYU1pYQ193bVEgICAxICAgMCAgICAgICAgICA4ICAgICAgICAgICAgMCAgICAgNDQuN2tiICAgICAgICAgNDQuN2tiCg==" + }, + "targetRequest": { + "Request-URI": "/_cat/indices?v", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "Authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "User-Agent": "curl/8.5.0", + "Accept": "*/*", + "body": "" + }, + "targetResponses": [ + { + "HTTP-Version": { + "keepAliveDefault": true + }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 721, + "content-type": "text/plain; charset=UTF-8", + "content-length": "484", + "body": "aGVhbHRoIHN0YXR1cyBpbmRleCAgICAgICAgICAgICAgICAgICAgIHV1aWQgICAgICAgICAgICAgICAgICAgcHJpIHJlcCBkb2NzLmNvdW50IGRvY3MuZGVsZXRlZCBzdG9yZS5zaXplIHByaS5zdG9yZS5zaXplCmdyZWVuICBvcGVuICAgLm9wZW5zZWFyY2gtb2JzZXJ2YWJpbGl0eSA4Vy1vWUhmYlN5U3JkeFFFX3NPbnpnICAgMSAgIDAgICAgICAgICAgMCAgICAgICAgICAgIDAgICAgICAgMjA4YiAgICAgICAgICAgMjA4YgpncmVlbiAgb3BlbiAgIC5wbHVnaW5zLW1sLWNvbmZpZyAgICAgICAgRjludnh2c2dSelNibG1mSnZ2aGptdyAgIDEgICAwICAgICAgICAgIDEgICAgICAgICAgICAwICAgICAgMy44a2IgICAgICAgICAgMy44a2IKZ3JlZW4gIG9wZW4gICAub3BlbmRpc3Ryb19zZWN1cml0eSAgICAgIDVmWHlhbkZuU2tDUUQ2bjFKUW1KTlEgICAxICAgMCAgICAgICAgIDEwICAgICAgICAgICAgMCAgICAgNzcuNWtiICAgICAgICAgNzcuNWtiCg==" + } + ], + "connectionId": "0242acfffe13000a-0000000a-00000005-1eb087a9beb83f3e-a32794b4.0", + "numRequests": 1, + "numErrors": 0 +} diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/valid_tuple_parsed.json b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/valid_tuple_parsed.json new file mode 100644 index 000000000..c587208ab --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/valid_tuple_parsed.json @@ -0,0 +1,49 @@ +{ + "sourceRequest": { + "Request-URI": "/_cat/indices?v", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "Authorization": "Basic YWRtaW46YWRtaW4=", + "User-Agent": "curl/8.5.0", + "Accept": "*/*", + "body": "" + }, + "sourceResponse": { + "HTTP-Version": { + "keepAliveDefault": true + }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 59, + "content-type": "text/plain; charset=UTF-8", + "content-length": "214", + "body": "health status index uuid pri rep docs.count docs.deleted store.size pri.store.size\ngreen open searchguard yJKXPMHtTRONIXSZXC_wmQ 1 0 8 0 44.7kb 44.7kb\n" + }, + "targetRequest": { + "Request-URI": "/_cat/indices?v", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "Authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "User-Agent": "curl/8.5.0", + "Accept": "*/*", + "body": "" + }, + "targetResponses": [ + { + "HTTP-Version": { + "keepAliveDefault": true + }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 721, + "content-type": "text/plain; charset=UTF-8", + "content-length": "484", + "body": "health status index uuid pri rep docs.count docs.deleted store.size pri.store.size\ngreen open .opensearch-observability 8W-oYHfbSySrdxQE_sOnzg 1 0 0 0 208b 208b\ngreen open .plugins-ml-config F9nvxvsgRzSblmfJvvhjmw 1 0 1 0 3.8kb 3.8kb\ngreen open .opendistro_security 5fXyanFnSkCQD6n1JQmJNQ 1 0 10 0 77.5kb 77.5kb\n" + } + ], + "connectionId": "0242acfffe13000a-0000000a-00000005-1eb087a9beb83f3e-a32794b4.0", + "numRequests": 1, + "numErrors": 0 +} \ No newline at end of file diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_cli.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_cli.py index 6ace00371..51305fb5a 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_cli.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_cli.py @@ -608,3 +608,33 @@ def test_cli_kafka_describe_topic(runner, mocker): def test_completion_script(runner): result = runner.invoke(cli, [str(VALID_SERVICES_YAML), 'completion', 'bash'], catch_exceptions=True) assert result.exit_code == 0 + + +def test_tuple_converter(runner, tmp_path): + # The `multiple_tuples` and `multiple_tuples_parsed` files are formatted as "real" json objects so that + # they can be pretty-printed and examined, but they need to be converted to ndjson files to be used by the + # CLI command. + + # Make the input file + input_tuples_file = f"{TEST_DATA_DIRECTORY}/multiple_tuples.json" + with open(input_tuples_file, 'r') as f: + input_tuples = json.load(f) + ndjson_input_file = f"{tmp_path}/tuples.ndjson" + with open(ndjson_input_file, 'w') as f: + f.write('\n'.join([json.dumps(record) for record in input_tuples])) + + ndjson_output_file = f"{tmp_path}/converted_tuples.ndjson" + result = runner.invoke(cli, ['--config-file', str(VALID_SERVICES_YAML), 'tuples', 'show', + '--in', ndjson_input_file, + '--out', ndjson_output_file], + catch_exceptions=True) + assert ndjson_output_file in result.output + assert result.exit_code == 0 + + # Open the ndjson output file and compare it to the "real" output file + expected_output_file = f"{TEST_DATA_DIRECTORY}/multiple_tuples_parsed.json" + with open(expected_output_file, 'r') as f: + output_tuples = json.load(f) + expected_output_as_ndjson = [json.dumps(record) + "\n" for record in output_tuples] + + assert open(ndjson_output_file).readlines() == expected_output_as_ndjson diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_tuple_reader.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_tuple_reader.py new file mode 100644 index 000000000..7f24cc49e --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_tuple_reader.py @@ -0,0 +1,163 @@ +import json +import pathlib +import re + +import pytest + +from console_link.models.tuple_reader import (DictionaryPathException, get_element_with_regex, get_element, + set_element, Flag, get_flags_for_component, + parse_tuple) + + +TEST_DATA_DIRECTORY = pathlib.Path(__file__).parent / "data" +VALID_TUPLE = TEST_DATA_DIRECTORY / "valid_tuple.json" +VALID_TUPLE_PARSED = TEST_DATA_DIRECTORY / "valid_tuple_parsed.json" +VALID_TUPLE_GZIPPED_CHUNKED = TEST_DATA_DIRECTORY / "valid_tuple_gzipped_and_chunked.json" +VALID_TUPLE_GZIPPED_CHUNKED_PARSED = TEST_DATA_DIRECTORY / "valid_tuple_gzipped_and_chunked_parsed.json" +INVALID_TUPLE = TEST_DATA_DIRECTORY / "invalid_tuple.json" + + +def test_get_element_with_regex_succeeds(): + d = { + 'A1': 'value', + '2B': 'not value' + } + regex = re.compile(r"\w\d") + assert get_element_with_regex(regex, d) == "value" + + +def test_get_element_with_regex_fails_no_raise(): + d = { + 'AA1': 'value', + '2B': 'not value' + } + regex = re.compile(r"\w\d") + assert get_element_with_regex(regex, d) is None + + +def test_get_element_with_regex_fails_with_raise(): + d = { + 'AA1': 'value', + '2B': 'not value' + } + regex = re.compile(r"\w\d") + with pytest.raises(DictionaryPathException): + get_element_with_regex(regex, d, raise_on_error=True) + + +def test_get_element_succeeds(): + d = { + 'A': { + 'B': 'value', + 'C': 'not value' + }, + 'B': 'not value' + } + assert get_element('A.B', d) == 'value' + + +def test_get_element_fails_no_raise(): + d = { + 'A': { + 'B': 'value', + 'C': 'not value' + }, + 'B': 'not value' + } + assert get_element('B.A', d) is None + + +def test_get_element_fails_raises(): + d = { + 'A': { + 'B': 'value', + 'C': 'not value' + }, + 'B': 'not value' + } + with pytest.raises(DictionaryPathException): + get_element('B.A', d, raise_on_error=True) + + +def test_set_element_succeeds(): + d = { + 'A': { + 'B': 'value', + 'C': 'not value' + }, + 'B': 'not value' + } + set_element('A.B', d, 'new value') + + assert d['A']['B'] == 'new value' + + +def test_set_element_fails_and_raises(): + d = { + 'A': { + 'B': 'value', + 'C': 'not value' + }, + 'B': 'not value' + } + with pytest.raises(DictionaryPathException): + set_element('B.A', d, 'new value') + + +def test_get_flags_none(): + request = { + 'Content-Encoding': 'not-gzip', + 'Content-Type': 'not-json', + 'Transfer-Encoding': 'not-chunked', + 'body': 'abcdefg' + } + flags = get_flags_for_component(request, False) + assert flags == set() + + +def test_get_flags_for_component_only_bulk(): + request = { + 'Content-Type': 'not-json', + 'body': 'abcdefg' + } + flags = get_flags_for_component(request, True) + assert flags == {Flag.Bulk_Request} + + +def test_get_flags_for_component_all_present(): + request = { + 'Content-Type': 'application/json', + 'body': 'abcdefg' + } + flags = get_flags_for_component(request, True) + assert flags == {Flag.Bulk_Request, Flag.Json} + + +def test_get_flags_all_present_alternate_capitalization(): + request = { + 'CONTENT-TYPE': 'application/json', + 'body': 'abcdefg' + } + flags = get_flags_for_component(request, True) + assert flags == {Flag.Bulk_Request, Flag.Json} + + +def test_parse_tuple_full_example(): + with open(VALID_TUPLE, 'r') as f: + tuple_ = f.read() + parsed = parse_tuple(tuple_, 0) + + with open(VALID_TUPLE_PARSED, 'r') as f: + expected = json.load(f) + + assert parsed == expected + + +def test_parse_tuple_with_malformed_bodies(caplog): + with open(INVALID_TUPLE, 'r') as f: + tuple_ = f.read() + + parsed = parse_tuple(tuple_, 0) + assert json.loads(tuple_) == parsed # Values weren't changed if they couldn't be interpreted + assert "Body value of sourceResponse on line 0 could not be decoded to utf-8" in caplog.text + assert "Body value of targetResponses item 0 on line 0 should be a json, but could not be parsed" in caplog.text diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java index 4d4981246..ab5287064 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java @@ -205,9 +205,7 @@ protected static Settings getSettings(@NonNull String configFile) { .entrySet().stream() .filter(kvp -> kvp.getKey().startsWith(HTTPS_CONFIG_PREFIX)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - if (!httpsSettings.containsKey(SUPPORTED_TLS_PROTOCOLS_LIST_KEY)) { - httpsSettings.put(SUPPORTED_TLS_PROTOCOLS_LIST_KEY, List.of("TLSv1.2", "TLSv1.3")); - } + httpsSettings.putIfAbsent(SUPPORTED_TLS_PROTOCOLS_LIST_KEY, List.of("TLSv1.2", "TLSv1.3")); return Settings.builder().loadFromMap(httpsSettings) // Don't bother with configurations the 'transport' (port 9300), which the plugin that we're using @@ -299,7 +297,7 @@ static Properties buildKafkaProperties(Parameters params) throws IOException { return kafkaProps; } - protected static IConnectionCaptureFactory getConnectionCaptureFactory( + protected static IConnectionCaptureFactory getConnectionCaptureFactory( Parameters params, RootCaptureContext rootContext ) throws IOException { @@ -433,12 +431,12 @@ public static void main(String[] args) throws InterruptedException, IOException proxy.waitForClose(); } - static ProxyChannelInitializer buildProxyChannelInitializer(RootCaptureContext rootContext, + static ProxyChannelInitializer buildProxyChannelInitializer(RootCaptureContext rootContext, BacksideConnectionPool backsideConnectionPool, Supplier sslEngineSupplier, @NonNull RequestCapturePredicate headerCapturePredicate, List headerOverridesArgs, - IConnectionCaptureFactory connectionFactory) + IConnectionCaptureFactory connectionFactory) { var headers = new ArrayList<>(convertPairListToMap(headerOverridesArgs).entrySet()); Collections.reverse(headers); @@ -453,7 +451,7 @@ static ProxyChannelInitializer buildProxyChannelInitializer(RootCaptureContext r removeStrings.add(kvp.getKey() + ":"); } - return new ProxyChannelInitializer( + return new ProxyChannelInitializer<>( rootContext, backsideConnectionPool, sslEngineSupplier, @@ -464,19 +462,16 @@ static ProxyChannelInitializer buildProxyChannelInitializer(RootCaptureContext r protected void initChannel(@NonNull SocketChannel ch) throws IOException { super.initChannel(ch); final var pipeline = ch.pipeline(); - { - int i = 0; - for (var kvp : headers) { - pipeline.addAfter(ProxyChannelInitializer.CAPTURE_HANDLER_NAME, "AddHeader-" + kvp.getKey(), - new HeaderAdderHandler(addBufs.get(i++))); - } + int i = 0; + for (var kvp : headers) { + pipeline.addAfter(ProxyChannelInitializer.CAPTURE_HANDLER_NAME, "AddHeader-" + kvp.getKey(), + new HeaderAdderHandler(addBufs.get(i++))); } - { - int i = 0; - for (var kvp : headers) { - pipeline.addAfter(ProxyChannelInitializer.CAPTURE_HANDLER_NAME, "RemoveHeader-" + kvp.getKey(), - new HeaderRemoverHandler(removeStrings.get(i++))); - } + + i = 0; + for (var kvp : headers) { + pipeline.addAfter(ProxyChannelInitializer.CAPTURE_HANDLER_NAME, "RemoveHeader-" + kvp.getKey(), + new HeaderRemoverHandler(removeStrings.get(i++))); } } }; diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPool.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPool.java index e79ec1212..f35c632e8 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPool.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/ExpiringSubstitutableItemPool.java @@ -234,16 +234,15 @@ public F getAvailableOrNewItem() { throw new PoolClosedException(); } var startTime = Instant.now(); - { - log.trace("getAvailableOrNewItem: readyItems.size()=" + readyItems.size()); - var item = readyItems.poll(); - log.trace("getAvailableOrNewItem: item=" + item + " remaining readyItems.size()=" + readyItems.size()); - if (item != null) { - stats.addHotGet(); - beginLoadingNewItemIfNecessary(); - stats.addWaitTime(Duration.between(startTime, Instant.now())); - return item.future; - } + log.atTrace().setMessage("getAvailableOrNewItem: readyItems.size()={}").addArgument(readyItems.size()).log(); + var item = readyItems.poll(); + log.atTrace().setMessage("getAvailableOrNewItem: item={} remaining readyItems.size()={}") + .addArgument(item).addArgument(readyItems.size()).log(); + if (item != null) { + stats.addHotGet(); + beginLoadingNewItemIfNecessary(); + stats.addWaitTime(Duration.between(startTime, Instant.now())); + return item.future; } BiFunction durationTrackingDecoratedItem = (itemsFuture, label) -> (F) itemsFuture.addListener( diff --git a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/NettyScanningHttpProxy.java b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/NettyScanningHttpProxy.java index 9ed135890..1dbdb00dc 100644 --- a/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/NettyScanningHttpProxy.java +++ b/TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/netty/NettyScanningHttpProxy.java @@ -22,7 +22,7 @@ public NettyScanningHttpProxy(int proxyPort) { this.proxyPort = proxyPort; } - public void start(ProxyChannelInitializer proxyChannelInitializer, int numThreads) + public void start(ProxyChannelInitializer proxyChannelInitializer, int numThreads) throws InterruptedException { bossGroup = new NioEventLoopGroup(1, new DefaultThreadFactory("captureProxyPoolBoss")); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpMessageAndTimestamp.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpMessageAndTimestamp.java index f0f9e9366..3251717b5 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpMessageAndTimestamp.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpMessageAndTimestamp.java @@ -61,9 +61,7 @@ public ByteBuf asByteBuf() { var compositeBuf = Unpooled.compositeBuffer(); packetBytes.stream() .map(Unpooled::wrappedBuffer) - .forEach(buffer -> { - compositeBuf.addComponent(true, buffer); - }); + .forEach(buffer -> compositeBuf.addComponent(true, buffer)); return compositeBuf.asReadOnly(); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ParsedHttpMessagesAsDicts.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ParsedHttpMessagesAsDicts.java index 34b20a237..7962c5000 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ParsedHttpMessagesAsDicts.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ParsedHttpMessagesAsDicts.java @@ -35,6 +35,7 @@ public class ParsedHttpMessagesAsDicts { public static final String STATUS_CODE_KEY = "Status-Code"; public static final String RESPONSE_TIME_MS_KEY = "response_time_ms"; public static final int MAX_PAYLOAD_BYTES_TO_CAPTURE = 256 * 1024 * 1024; + public static final String EXCEPTION_KEY_STRING = "Exception"; public final Optional> sourceRequestOp; public final Optional> sourceResponseOp; @@ -160,7 +161,7 @@ private static Map makeSafeMap( ) .setCause(e) .log(); - return Map.of("Exception", (Object) e.toString()); + return Map.of(EXCEPTION_KEY_STRING, (Object) e.toString()); } } @@ -185,7 +186,7 @@ private static Map convertRequest( context.setHttpVersion(message.protocolVersion().toString()); return fillMap(map, message.headers(), message.content()); } else { - return Map.of("Exception", "Message couldn't be parsed as a full http message"); + return Map.of(EXCEPTION_KEY_STRING, "Message couldn't be parsed as a full http message"); } } }); @@ -211,7 +212,7 @@ private static Map convertResponse( map.put(RESPONSE_TIME_MS_KEY, latency.toMillis()); return fillMap(map, message.headers(), message.content()); } else { - return Map.of("Exception", "Message couldn't be parsed as a full http message"); + return Map.of(EXCEPTION_KEY_STRING, "Message couldn't be parsed as a full http message"); } } }); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java index 084aa2801..45f8f732d 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayer.java @@ -52,7 +52,7 @@ public class TrafficReplayer { public static final String PACKET_TIMEOUT_SECONDS_PARAMETER_NAME = "--packet-timeout-seconds"; public static final String LOOKAHEAD_TIME_WINDOW_PARAMETER_NAME = "--lookahead-time-window"; - private static final long ACTIVE_WORK_MONITOR_CADENCE_MS = 30 * 1000; + private static final long ACTIVE_WORK_MONITOR_CADENCE_MS = 30 * 1000L; public static class DualException extends Exception { public final Throwable originalCause; @@ -357,7 +357,8 @@ public static void main(String[] args) throws Exception { params, Duration.ofSeconds(params.lookaheadTimeSeconds) ); - var authTransformer = buildAuthTransformerFactory(params) + var authTransformer = buildAuthTransformerFactory(params); + var trafficStreamLimiter = new TrafficStreamLimiter(params.maxConcurrentRequests) ) { var timeShifter = new TimeShifter(params.speedupFactor); var serverTimeout = Duration.ofSeconds(params.targetServerResponseTimeoutSeconds); @@ -379,7 +380,7 @@ public static void main(String[] args) throws Exception { params.allowInsecureConnections, params.numClientThreads ), - new TrafficStreamLimiter(params.maxConcurrentRequests), + trafficStreamLimiter, orderedRequestTracker ); activeContextMonitor = new ActiveContextMonitor( @@ -429,20 +430,20 @@ private static void setupShutdownHookForReplayer(TrafficReplayerTopLevel tr) { // both Log4J and the java builtin loggers add shutdown hooks. // The API for addShutdownHook says that those hooks registered will run in an undetermined order. // Hence, the reason that this code logs via slf4j logging AND stderr. - { - var beforeMsg = "Running TrafficReplayer Shutdown. " + Optional.of("Running TrafficReplayer Shutdown. " + "The logging facilities may also be shutting down concurrently, " - + "resulting in missing logs messages."; - log.atWarn().setMessage(beforeMsg).log(); - System.err.println(beforeMsg); - } + + "resulting in missing logs messages.") + .ifPresent(beforeMsg -> { + log.atWarn().setMessage(beforeMsg).log(); + System.err.println(beforeMsg); + }); Optional.ofNullable(weakTrafficReplayer.get()).ifPresent(o -> o.shutdown(null)); - { - var afterMsg = "Done shutting down TrafficReplayer (due to Runtime shutdown). " - + "Logs may be missing for events that have happened after the Shutdown event was received."; - log.atWarn().setMessage(afterMsg).log(); - System.err.println(afterMsg); - } + Optional.of("Done shutting down TrafficReplayer (due to Runtime shutdown). " + + "Logs may be missing for events that have happened after the Shutdown event was received.") + .ifPresent(afterMsg -> { + log.atWarn().setMessage(afterMsg).log(); + System.err.println(afterMsg); + }); })); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayerCore.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayerCore.java index 42788b4a8..4acf7d6ae 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayerCore.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayerCore.java @@ -281,7 +281,6 @@ private void packageAndWriteResponse( ) { log.trace("done sending and finalizing data to the packet handler"); - SourceTargetCaptureTuple requestResponseTuple1; if (t != null) { log.error("Got exception in CompletableFuture callback: ", t); } @@ -318,6 +317,7 @@ public TrackedFuture transformA request.packetBytes::stream); } + @Override protected void perResponseConsumer(AggregatedRawResponse summary, HttpRequestTransformationStatus transformationStatus, IReplayContexts.IReplayerHttpTransactionContext context) { @@ -330,7 +330,7 @@ protected void perResponseConsumer(AggregatedRawResponse summary, exceptionRequestCount.incrementAndGet(); } else if (transformationStatus.isError()) { log.atInfo() - .setCause(summary.getError()) + .setCause(Optional.ofNullable(summary).map(AggregatedRawResponse::getError).orElse(null)) .setMessage("Unknown error transforming {}: ") .addArgument(context) .log(); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayerTopLevel.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayerTopLevel.java index 9f21db0b4..7b0c32037 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayerTopLevel.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/TrafficReplayerTopLevel.java @@ -177,6 +177,16 @@ public void setupRunAndWaitForReplayToFinish( } } + /** + * Called after the TrafficReplayer has finished accumulating and reconstructing every transaction from + * the incoming stream. This implementation will NOT wait for the ReplayEngine independently to complete, + * but rather call waitForRemainingWork. If a subclass wants more details from either of the two main + * non-field components of a TrafficReplayer, they have access to each of them here. + * + * @param replayEngine The ReplayEngine that may still be working to send the accumulated requests. + * @param trafficToHttpTransactionAccumulator The accumulator that had reconstructed the incoming records and + * has now finished + */ protected void wrapUpWorkAndEmitSummary( ReplayEngine replayEngine, CapturedTrafficToHttpTransactionAccumulator trafficToHttpTransactionAccumulator diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java index 4527d1f5d..57fa4a4a3 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/NettyPacketToHttpConsumer.java @@ -167,9 +167,7 @@ public IReplayContexts.ITargetRequestContext getParentContext() { return (eventLoop, ctx) -> NettyPacketToHttpConsumer.createClientConnection(eventLoop, sslContext, uri, ctx); } - public static class ChannelNotActiveException extends IOException { - public ChannelNotActiveException() {} - } + public static class ChannelNotActiveException extends IOException { } public static TrackedFuture createClientConnection( EventLoop eventLoop, @@ -382,9 +380,8 @@ public TrackedFuture consumeBytes(ByteBuf packetData) { + packetData.toString(StandardCharsets.UTF_8) ) .log(); - return writePacketAndUpdateFuture(packetData).whenComplete((v2, t2) -> { - log.atTrace().setMessage(() -> "finished writing " + httpContext() + " t=" + t2).log(); - }, () -> ""); + return writePacketAndUpdateFuture(packetData).whenComplete((v2, t2) -> + log.atTrace().setMessage(() -> "finished writing " + httpContext() + " t=" + t2).log(), () -> ""); } else { log.atWarn() .setMessage( diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java index 763cbd868..ff8c3f665 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java @@ -70,6 +70,7 @@ public int size() { return underlyingMap.entrySet(); } } + @Override public Object put(String key, Object value) { return underlyingMap.put(key, value); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/http/retries/DefaultRetry.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/http/retries/DefaultRetry.java index 50a6ac6ef..c25dbe5dd 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/http/retries/DefaultRetry.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/http/retries/DefaultRetry.java @@ -71,7 +71,7 @@ public static boolean retryIsUnnecessaryGivenStatusCode(int statusCode) { Optional.ofNullable(HttpByteBufFormatter.processHttpMessageFromBufs( HttpByteBufFormatter.HttpMessageType.RESPONSE, Stream.of(sourceResponse.asByteBuf())))) - .filter(o -> o instanceof HttpResponse) + .filter(HttpResponse.class::isInstance) .map(responseMsg -> shouldRetry(((HttpResponse)responseMsg).status().code(), rr.status().code())) .orElse(RequestSenderOrchestrator.RetryDirective.RETRY), diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/http/retries/OpenSearchDefaultRetry.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/http/retries/OpenSearchDefaultRetry.java index b1039c777..47865ad0f 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/http/retries/OpenSearchDefaultRetry.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/http/retries/OpenSearchDefaultRetry.java @@ -30,7 +30,7 @@ @Slf4j public class OpenSearchDefaultRetry extends DefaultRetry { - private static final Pattern bulkPathMatcher = Pattern.compile("^(/[^/]*)?/_bulk([/?]+.*)*$"); + private static final Pattern bulkPathMatcher = Pattern.compile("^(/[^/]*)?/_bulk(/.*)?$"); private static class BulkErrorFindingHandler extends ChannelInboundHandlerAdapter { private final JsonParser parser; @@ -82,11 +82,12 @@ private void consumeInput() throws IOException { break; } else if (parser.getParsingContext().inRoot() && token == JsonToken.END_OBJECT) { break; - } else if (token != JsonToken.START_OBJECT && token != JsonToken.END_OBJECT) { + } else if (token != JsonToken.START_OBJECT && + token != JsonToken.END_OBJECT && + !parser.getParsingContext().inRoot()) + { // Skip non-root level content - if (!parser.getParsingContext().inRoot()) { - parser.skipChildren(); - } + parser.skipChildren(); } } } @@ -109,24 +110,27 @@ boolean bulkResponseHadNoErrors(ByteBuf responseByteBuf) { var targetRequestByteBuf = Unpooled.wrappedBuffer(targetRequestBytes); var parsedRequest = HttpByteBufFormatter.parseHttpRequestFromBufs(Stream.of(targetRequestByteBuf), 0); if (parsedRequest != null && - bulkPathMatcher.matcher(parsedRequest.uri()).matches()) { + bulkPathMatcher.matcher(parsedRequest.uri()).matches() && // do a more granular check. If the raw response wasn't present, then just push it to the superclass // since it isn't going to be any kind of response, let alone a bulk one - if (Optional.ofNullable(currentResponse.getRawResponse()).map(r->r.status().code() == 200).orElse(false)) { - if (bulkResponseHadNoErrors(currentResponse.getResponseAsByteBuf())) { - return TextTrackedFuture.completedFuture(RequestSenderOrchestrator.RetryDirective.DONE, - () -> "no errors found in the target response, so not retrying"); - } else { - return reconstructedSourceTransactionFuture.thenCompose(rrp -> - TextTrackedFuture.completedFuture( - bulkResponseHadNoErrors(rrp.getResponseData().asByteBuf()) ? - RequestSenderOrchestrator.RetryDirective.RETRY : - RequestSenderOrchestrator.RetryDirective.DONE, - () -> "evaluating retry status dependent upon source error field"), - () -> "checking the accumulated source response value"); - } + Optional.ofNullable(currentResponse.getRawResponse()) + .map(r->r.status().code() == 200) + .orElse(false)) + { + if (bulkResponseHadNoErrors(currentResponse.getResponseAsByteBuf())) { + return TextTrackedFuture.completedFuture(RequestSenderOrchestrator.RetryDirective.DONE, + () -> "no errors found in the target response, so not retrying"); + } else { + return reconstructedSourceTransactionFuture.thenCompose(rrp -> + TextTrackedFuture.completedFuture( + bulkResponseHadNoErrors(rrp.getResponseData().asByteBuf()) ? + RequestSenderOrchestrator.RetryDirective.RETRY : + RequestSenderOrchestrator.RetryDirective.DONE, + () -> "evaluating retry status dependent upon source error field"), + () -> "checking the accumulated source response value"); } } + return super.shouldRetry(targetRequestBytes, currentResponse, reconstructedSourceTransactionFuture); } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/OffsetLifecycleTracker.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/OffsetLifecycleTracker.java index a75888f4b..f96f02d4f 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/OffsetLifecycleTracker.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/kafka/OffsetLifecycleTracker.java @@ -42,15 +42,17 @@ void add(long offset) { Optional removeAndReturnNewHead(long offsetToRemove) { synchronized (pQueue) { var topCursor = pQueue.peek(); - assert topCursor != null : "Expected pQueue to be non-empty but it was when asked to remove " - + offsetToRemove; - var didRemove = pQueue.remove(offsetToRemove); - assert didRemove : "Expected all live records to have an entry and for them to be removed only once"; if (topCursor == null) { throw new IllegalStateException( "pQueue looks to have been empty by the time we tried to remove " + offsetToRemove ); } + var didRemove = pQueue.remove(offsetToRemove); + if (!didRemove) { + throw new IllegalStateException( + "Expected all live records to have an entry and for them to be removed only once"); + } + if (offsetToRemove == topCursor) { topCursor = Optional.ofNullable(pQueue.peek()).orElse(cursorHighWatermark + 1); // most recent cursor // was previously popped diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelContextManager.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelContextManager.java index ab31b450c..711b1d1de 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelContextManager.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/ChannelContextManager.java @@ -57,7 +57,7 @@ public IReplayContexts.IChannelKeyContext retainOrCreateContext(ITrafficStreamKe public IReplayContexts.IChannelKeyContext releaseContextFor(IReplayContexts.IChannelKeyContext ctx) { var connId = ctx.getConnectionId(); var refCountedCtx = connectionToChannelContextMap.get(connId); - assert ctx == refCountedCtx.context; + assert ctx == refCountedCtx.context : "consistency mismatch"; var finalRelease = refCountedCtx.release(); if (finalRelease) { ctx.close(); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/KafkaConsumerContexts.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/KafkaConsumerContexts.java index 024e1e2c4..c1f97580d 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/KafkaConsumerContexts.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/tracing/KafkaConsumerContexts.java @@ -24,7 +24,7 @@ private KafkaConsumerContexts() {} public static class AsyncListeningContext implements IKafkaConsumerContexts.IAsyncListeningContext { @Getter @NonNull - public final RootReplayerContext enclosingScope; // TODO - rename this to rootScope + public final RootReplayerContext enclosingScope; @Getter @Setter Exception observedExceptionToIncludeInMetrics; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/util/RefSafeHolder.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/util/RefSafeHolder.java index a67a3602e..a9a4d4ef6 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/util/RefSafeHolder.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/util/RefSafeHolder.java @@ -14,7 +14,7 @@ private RefSafeHolder(@Nullable T resource) { } @MustBeClosed - static public RefSafeHolder create(@Nullable T resource) { + public static RefSafeHolder create(@Nullable T resource) { return new RefSafeHolder<>(resource); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/trafficcapture/protos/TrafficStreamUtils.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/trafficcapture/protos/TrafficStreamUtils.java index 096082467..5f942e64b 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/trafficcapture/protos/TrafficStreamUtils.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/trafficcapture/protos/TrafficStreamUtils.java @@ -35,7 +35,7 @@ public static String summarizeTrafficStream(TrafficStream ts) { return ts.getConnectionId() + " (#" + getTrafficStreamIndex(ts) + ")[" + listSummaryStr + "]"; } - private static Object getOptionalContext(TrafficObservation tso) { + private static String getOptionalContext(TrafficObservation tso) { return Optional.ofNullable(getByteArrayForDataOf(tso)) .map(b -> " " + new String(b, 0, Math.min(3, b.length), StandardCharsets.UTF_8)) .orElse(""); diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/StaticAuthTransformerFactory.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/StaticAuthTransformerFactory.java index 7e319f495..dfdae7de0 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/StaticAuthTransformerFactory.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/transform/StaticAuthTransformerFactory.java @@ -15,7 +15,6 @@ public IAuthTransformer getAuthTransformer(HttpJsonMessageWithFaultingPayload ht @Override public void rewriteHeaders(HttpJsonMessageWithFaultingPayload msg) { msg.headers().put("authorization", authHeaderValue); - // TODO - wipe out more headers too? } }; } diff --git a/deployment/cdk/opensearch-service-migration/bin/app.ts b/deployment/cdk/opensearch-service-migration/bin/app.ts index c632b6301..1af7660f8 100644 --- a/deployment/cdk/opensearch-service-migration/bin/app.ts +++ b/deployment/cdk/opensearch-service-migration/bin/app.ts @@ -1,34 +1,6 @@ #!/usr/bin/env node import 'source-map-support/register'; -import {readFileSync} from 'fs'; -import {App, Tags} from 'aws-cdk-lib'; -import {StackComposer} from "../lib/stack-composer"; +import { createApp } from './createApp'; -const app = new App(); -const versionFile = readFileSync('../../../VERSION', 'utf-8') -// Remove any blank newlines because this would be an invalid tag value -const version = versionFile.replace(/\n/g, ''); -Tags.of(app).add("migration_deployment", version) -const account = process.env.CDK_DEFAULT_ACCOUNT -const region = process.env.CDK_DEFAULT_REGION -// Environment setting to allow providing an existing AWS AppRegistry application ARN which each created CDK stack -// from this CDK app will be added to. -const migrationsAppRegistryARN = process.env.MIGRATIONS_APP_REGISTRY_ARN -if (migrationsAppRegistryARN) { - console.info(`App Registry mode is enabled for CFN stack tracking. Will attempt to import the App Registry application from the MIGRATIONS_APP_REGISTRY_ARN env variable of ${migrationsAppRegistryARN} and looking in the configured region of ${region}`) -} - -// Temporarily allow both means for providing an additional migrations User Agent, but remove CUSTOM_REPLAYER_USER_AGENT -// in future change -let migrationsUserAgent = undefined -if (process.env.CUSTOM_REPLAYER_USER_AGENT) - migrationsUserAgent = process.env.CUSTOM_REPLAYER_USER_AGENT -if (process.env.MIGRATIONS_USER_AGENT) - migrationsUserAgent = process.env.MIGRATIONS_USER_AGENT - -new StackComposer(app, { - migrationsAppRegistryARN: migrationsAppRegistryARN, - migrationsUserAgent: migrationsUserAgent, - migrationsSolutionVersion: version, - env: { account: account, region: region } -}); +const app = createApp(); +app.synth(); diff --git a/deployment/cdk/opensearch-service-migration/bin/createApp.ts b/deployment/cdk/opensearch-service-migration/bin/createApp.ts new file mode 100644 index 000000000..69440e010 --- /dev/null +++ b/deployment/cdk/opensearch-service-migration/bin/createApp.ts @@ -0,0 +1,28 @@ +import { App, Tags } from 'aws-cdk-lib'; +import { readFileSync } from 'fs'; +import { StackComposer } from "../lib/stack-composer"; + +export function createApp(): App { + const app = new App(); + const versionFile = readFileSync('../../../VERSION', 'utf-8'); + const version = versionFile.replace(/\n/g, ''); + Tags.of(app).add("migration_deployment", version); + + const account = process.env.CDK_DEFAULT_ACCOUNT; + const region = process.env.CDK_DEFAULT_REGION; + const migrationsAppRegistryARN = process.env.MIGRATIONS_APP_REGISTRY_ARN; + const customReplayerUserAgent = process.env.CUSTOM_REPLAYER_USER_AGENT; + + if (migrationsAppRegistryARN) { + console.info(`App Registry mode is enabled for CFN stack tracking. Will attempt to import the App Registry application from the MIGRATIONS_APP_REGISTRY_ARN env variable of ${migrationsAppRegistryARN} and looking in the configured region of ${region}`); + } + + new StackComposer(app, { + migrationsAppRegistryARN: migrationsAppRegistryARN, + customReplayerUserAgent: customReplayerUserAgent, + migrationsSolutionVersion: version, + env: { account: account, region: region } + }); + + return app; +} diff --git a/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts b/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts index 2ce5358bd..452b59d6d 100644 --- a/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts +++ b/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts @@ -2,8 +2,6 @@ import {Effect, PolicyStatement, Role, ServicePrincipal} from "aws-cdk-lib/aws-i import {Construct} from "constructs"; import {CpuArchitecture} from "aws-cdk-lib/aws-ecs"; import {RemovalPolicy} from "aws-cdk-lib"; -import { IApplicationLoadBalancer } from "aws-cdk-lib/aws-elasticloadbalancingv2"; -import { ICertificate } from "aws-cdk-lib/aws-certificatemanager"; import { IStringParameter, StringParameter } from "aws-cdk-lib/aws-ssm"; import * as forge from 'node-forge'; import * as yargs from 'yargs'; @@ -211,20 +209,18 @@ export function createDefaultECSTaskRole(scope: Construct, serviceName: string): } export function validateFargateCpuArch(cpuArch?: string): CpuArchitecture { - const desiredArch = cpuArch ? cpuArch : process.arch + const desiredArch = cpuArch ?? process.arch const desiredArchUpper = desiredArch.toUpperCase() if (desiredArchUpper === "X86_64" || desiredArchUpper === "X64") { return CpuArchitecture.X86_64 } else if (desiredArchUpper === "ARM64") { return CpuArchitecture.ARM64 - } else { - if (cpuArch) { - throw new Error(`Unknown Fargate cpu architecture provided: ${desiredArch}`) - } - else { - throw new Error(`Unsupported process cpu architecture detected: ${desiredArch}, CDK requires X64 or ARM64 for Docker image compatability`) - } + } else if (cpuArch) { + throw new Error(`Unknown Fargate cpu architecture provided: ${desiredArch}`) + } + else { + throw new Error(`Unsupported process cpu architecture detected: ${desiredArch}, CDK requires X64 or ARM64 for Docker image compatability`) } } @@ -235,21 +231,6 @@ export function parseRemovalPolicy(optionName: string, policyNameString?: string } return policy } - - -export type ALBConfig = NewALBListenerConfig; - -export interface NewALBListenerConfig { - alb: IApplicationLoadBalancer, - albListenerCert: ICertificate, - albListenerPort?: number, -} - -export function isNewALBListenerConfig(config: ALBConfig): config is NewALBListenerConfig { - const parsed = config as NewALBListenerConfig; - return parsed.alb !== undefined && parsed.albListenerCert !== undefined; -} - export function hashStringSHA256(message: string): string { const md = forge.md.sha256.create(); md.update(message); @@ -317,7 +298,7 @@ export enum MigrationSSMParameter { } -export class ClusterNoAuth {}; +export class ClusterNoAuth {} export class ClusterSigV4Auth { region?: string; @@ -443,4 +424,4 @@ export function parseClusterDefinition(json: any): ClusterYaml { throw new Error(`Invalid auth type when parsing cluster definition: ${json.auth.type}`) } return new ClusterYaml({endpoint, version, auth}) -} \ No newline at end of file +} diff --git a/deployment/cdk/opensearch-service-migration/lib/lambda/acm-cert-importer-handler.ts b/deployment/cdk/opensearch-service-migration/lib/lambda/acm-cert-importer-handler.ts index 4763f960d..1cf7157e3 100644 --- a/deployment/cdk/opensearch-service-migration/lib/lambda/acm-cert-importer-handler.ts +++ b/deployment/cdk/opensearch-service-migration/lib/lambda/acm-cert-importer-handler.ts @@ -15,23 +15,26 @@ export const handler = async (event: CloudFormationCustomResourceEvent, context: try { switch (event.RequestType) { - case 'Create': + case 'Create': { const { certificate, privateKey, certificateChain } = await generateSelfSignedCertificate(); const certificateArn = await importCertificate(certificate, privateKey, certificateChain); console.log(`Certificate imported with ARN: ${certificateArn}`); responseData = { CertificateArn: certificateArn }; physicalResourceId = certificateArn; break; - case 'Update': + } + case 'Update': { // No update logic needed, return existing physical resource id physicalResourceId = event.PhysicalResourceId; - break; - case 'Delete': + break; + } + case 'Delete': { const arn = event.PhysicalResourceId; await deleteCertificate(arn); responseData = { CertificateArn: arn }; physicalResourceId = arn; break; + } } return await sendResponse(event, context, 'SUCCESS', responseData, physicalResourceId); @@ -45,7 +48,7 @@ async function generateSelfSignedCertificate(): Promise<{ certificate: string, p return new Promise((resolve, reject) => { const keys = forge.pki.rsa.generateKeyPair(2048); const cert = forge.pki.createCertificate(); - + cert.publicKey = keys.publicKey; cert.serialNumber = '01'; cert.validity.notBefore = new Date(Date.UTC(1970, 0, 1, 0, 0, 0)); @@ -54,10 +57,10 @@ async function generateSelfSignedCertificate(): Promise<{ certificate: string, p name: 'commonName', value: 'localhost' }]; - + cert.setSubject(attrs); cert.setIssuer(attrs); - + cert.setExtensions([{ name: 'basicConstraints', cA: true @@ -78,7 +81,7 @@ async function generateSelfSignedCertificate(): Promise<{ certificate: string, p clientAuth: true },]); cert.sign(keys.privateKey, forge.md.sha384.create()); - + const pemCert = forge.pki.certificateToPem(cert); const pemKey = forge.pki.privateKeyToPem(keys.privateKey); @@ -165,7 +168,7 @@ async function sendResponse(event: CloudFormationCustomResourceEvent, context: C }); }); - request.on('error', (error) => { + request.on('error', (error: Error) => { console.error('sendResponse Error:', error); reject(error); }); @@ -173,4 +176,4 @@ async function sendResponse(event: CloudFormationCustomResourceEvent, context: C request.write(responseBody); request.end(); }); -} \ No newline at end of file +} diff --git a/deployment/cdk/opensearch-service-migration/lib/migration-services-yaml.ts b/deployment/cdk/opensearch-service-migration/lib/migration-services-yaml.ts index a2677b67c..24d5020c7 100644 --- a/deployment/cdk/opensearch-service-migration/lib/migration-services-yaml.ts +++ b/deployment/cdk/opensearch-service-migration/lib/migration-services-yaml.ts @@ -110,13 +110,6 @@ export class MetadataMigrationYaml { otel_endpoint: string = ''; source_cluster_version?: string; } - -export class MSKYaml { -} - -export class StandardKafkaYaml { -} - export class KafkaYaml { broker_endpoints: string = ''; msk?: string | null; diff --git a/deployment/cdk/opensearch-service-migration/lib/network-stack.ts b/deployment/cdk/opensearch-service-migration/lib/network-stack.ts index 20f1939ba..579a284bd 100644 --- a/deployment/cdk/opensearch-service-migration/lib/network-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/network-stack.ts @@ -110,7 +110,7 @@ export class NetworkStack extends Stack { this.vpc = new Vpc(this, 'domainVPC', { // IP space should be customized for use cases that have specific IP range needs ipAddresses: IpAddresses.cidr('10.0.0.0/16'), - maxAzs: zoneCount ? zoneCount : 2, + maxAzs: zoneCount ?? 2, subnetConfiguration: [ // Outbound internet access for private subnets require a NAT Gateway which must live in // a public subnet diff --git a/deployment/cdk/opensearch-service-migration/lib/opensearch-domain-stack.ts b/deployment/cdk/opensearch-service-migration/lib/opensearch-domain-stack.ts index 84f9cbb8d..52210d277 100644 --- a/deployment/cdk/opensearch-service-migration/lib/opensearch-domain-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/opensearch-domain-stack.ts @@ -15,8 +15,14 @@ import {ILogGroup, LogGroup} from "aws-cdk-lib/aws-logs"; import {ISecret, Secret} from "aws-cdk-lib/aws-secretsmanager"; import {StackPropsExt} from "./stack-composer"; import { ClusterYaml } from "./migration-services-yaml"; -import { ClusterAuth, ClusterBasicAuth, ClusterNoAuth } from "./common-utilities" -import { MigrationSSMParameter, createMigrationStringParameter, getMigrationStringParameterValue } from "./common-utilities"; +import { + ClusterAuth, + ClusterBasicAuth, + ClusterNoAuth, + MigrationSSMParameter, + createMigrationStringParameter, + getMigrationStringParameterValue +} from "./common-utilities"; export interface OpensearchDomainStackProps extends StackPropsExt { @@ -56,8 +62,6 @@ export interface OpensearchDomainStackProps extends StackPropsExt { } -export const osClusterEndpointParameterName = "osClusterEndpoint"; - export class OpenSearchDomainStack extends Stack { targetClusterYaml: ClusterYaml; @@ -99,8 +103,7 @@ export class OpenSearchDomainStack extends Stack { } createSSMParameters(domain: Domain, adminUserName: string|undefined, adminUserSecret: ISecret|undefined, stage: string, deployId: string) { - const endpointParameter = osClusterEndpointParameterName - const endpointSSM = createMigrationStringParameter(this, `https://${domain.domainEndpoint}:443`, { + createMigrationStringParameter(this, `https://${domain.domainEndpoint}:443`, { parameter: MigrationSSMParameter.OS_CLUSTER_ENDPOINT, defaultDeployId: deployId, stage, @@ -108,10 +111,10 @@ export class OpenSearchDomainStack extends Stack { if (domain.masterUserPassword && !adminUserSecret) { console.log(`An OpenSearch domain fine-grained access control user was configured without an existing Secrets Manager secret, will not create SSM Parameter: /migration/${stage}/${deployId}/osUserAndSecret`) } else if (domain.masterUserPassword && adminUserSecret) { - const secretSSM = createMigrationStringParameter(this, `${adminUserName} ${adminUserSecret.secretArn}`, { + createMigrationStringParameter(this, `${adminUserName} ${adminUserSecret.secretArn}`, { parameter: MigrationSSMParameter.OS_USER_AND_SECRET_ARN, defaultDeployId: deployId, - stage, + stage, }); } } @@ -138,12 +141,11 @@ export class OpenSearchDomainStack extends Stack { const appLG: ILogGroup|undefined = props.appLogGroup && props.appLogEnabled ? LogGroup.fromLogGroupArn(this, "appLogGroup", props.appLogGroup) : undefined - const domainAccessSecurityGroupParameter = props.domainAccessSecurityGroupParameter ?? "osAccessSecurityGroupId" const defaultOSClusterAccessGroup = SecurityGroup.fromSecurityGroupId(this, "defaultDomainAccessSG", getMigrationStringParameterValue(this, { ...props, parameter: MigrationSSMParameter.OS_ACCESS_SECURITY_GROUP_ID, })); - + let adminUserSecret: ISecret|undefined = props.fineGrainedManagerUserSecretManagerKeyARN ? Secret.fromSecretCompleteArn(this, "managerSecret", props.fineGrainedManagerUserSecretManagerKeyARN) : undefined // Map objects from props diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts index d5eb90a21..49e57a212 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts @@ -89,7 +89,7 @@ export class CaptureProxyStack extends MigrationServiceCore { constructor(scope: Construct, id: string, props: CaptureProxyProps) { super(scope, id, props) - const serviceName = props.serviceName || "capture-proxy"; + const serviceName = props.serviceName ?? "capture-proxy"; let securityGroupConfigs = [ { id: "serviceSG", param: MigrationSSMParameter.SERVICE_SECURITY_GROUP_ID }, diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts index 96135d343..348f6fdcb 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts @@ -1,6 +1,6 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {CpuArchitecture, MountPoint, PortMapping, Protocol, Volume} from "aws-cdk-lib/aws-ecs"; +import {CpuArchitecture, PortMapping, Protocol} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {join} from "path"; import {Effect, PolicyStatement, Role, ServicePrincipal} from "aws-cdk-lib/aws-iam"; diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-otel-collector-sidecar.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-otel-collector-sidecar.ts index 029a20eaa..a23d44daa 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-otel-collector-sidecar.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-otel-collector-sidecar.ts @@ -13,8 +13,8 @@ import { DockerImageAsset } from "aws-cdk-lib/aws-ecr-assets"; import { join } from "path"; export class OtelCollectorSidecar { - public static OTEL_CONTAINER_PORT = 4317; - public static OTEL_CONTAINER_HEALTHCHECK_PORT = 13133; + public static readonly OTEL_CONTAINER_PORT = 4317; + public static readonly OTEL_CONTAINER_HEALTHCHECK_PORT = 13133; static getOtelLocalhostEndpoint() { return "http://localhost:" + OtelCollectorSidecar.OTEL_CONTAINER_PORT; diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts index 432e57c0c..945888162 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts @@ -1,7 +1,6 @@ import {StackPropsExt} from "../stack-composer"; import {ISecurityGroup, IVpc, SubnetType} from "aws-cdk-lib/aws-ec2"; import { - CfnService as FargateCfnService, Cluster, ContainerImage, CpuArchitecture, FargateService, @@ -140,8 +139,8 @@ export class MigrationServiceCore extends Stack { let startupPeriodSeconds = 30; // Add a separate container to monitor and fail healthcheck after a given maxUptime const maxUptimeContainer = serviceTaskDef.addContainer("MaxUptimeContainer", { - image: ContainerImage.fromRegistry("public.ecr.aws/amazonlinux/amazonlinux:2023-minimal"), - memoryLimitMiB: 64, + image: ContainerImage.fromRegistry("public.ecr.aws/amazonlinux/amazonlinux:2023-minimal"), + memoryLimitMiB: 64, entryPoint: [ "/bin/sh", "-c", @@ -185,10 +184,10 @@ export class MigrationServiceCore extends Stack { securityGroups: props.securityGroups, vpcSubnets: props.vpc.selectSubnets({subnetType: SubnetType.PRIVATE_WITH_EGRESS}), }); - + if (props.targetGroups) { props.targetGroups.filter(tg => tg !== undefined).forEach(tg => tg.addTarget(fargateService)); } } -} \ No newline at end of file +} diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/reindex-from-snapshot-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/reindex-from-snapshot-stack.ts index b42f3dde5..d453ebe77 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/reindex-from-snapshot-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/reindex-from-snapshot-stack.ts @@ -79,9 +79,9 @@ export class ReindexFromSnapshotStack extends MigrationServiceCore { let targetPassword = ""; let targetPasswordArn = ""; if (props.clusterAuthDetails.basicAuth) { - targetUser = props.clusterAuthDetails.basicAuth.username, - targetPassword = props.clusterAuthDetails.basicAuth.password || "", - targetPasswordArn = props.clusterAuthDetails.basicAuth.password_from_secret_arn || "" + targetUser = props.clusterAuthDetails.basicAuth.username + targetPassword = props.clusterAuthDetails.basicAuth.password ?? "" + targetPasswordArn = props.clusterAuthDetails.basicAuth.password_from_secret_arn ?? "" }; const sharedLogFileSystem = new SharedLogFileSystem(this, props.stage, props.defaultDeployId); const openSearchPolicy = createOpenSearchIAMAccessPolicy(this.partition, this.region, this.account); diff --git a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts index 6066ef26c..f6f3ae88b 100644 --- a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts +++ b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts @@ -86,10 +86,10 @@ export class StackComposer { private getEngineVersion(engineVersionString: string) : EngineVersion { let version: EngineVersion - if (engineVersionString && engineVersionString.startsWith("OS_")) { + if (engineVersionString?.startsWith("OS_")) { // Will accept a period delimited version string (i.e. 1.3) and return a proper EngineVersion version = EngineVersion.openSearch(engineVersionString.substring(3)) - } else if (engineVersionString && engineVersionString.startsWith("ES_")) { + } else if (engineVersionString?.startsWith("ES_")) { version = EngineVersion.elasticsearch(engineVersionString.substring(3)) } else { throw new Error(`Engine version (${engineVersionString}) is not present or does not match the expected format, i.e. OS_1.3 or ES_7.9`) @@ -231,7 +231,7 @@ export class StackComposer { "auth": {"type": "none"} } } - const sourceClusterDisabled = sourceClusterDefinition?.disabled ? true : false + const sourceClusterDisabled = !!sourceClusterDefinition?.disabled const sourceCluster = (sourceClusterDefinition && !sourceClusterDisabled) ? parseClusterDefinition(sourceClusterDefinition) : undefined const sourceClusterEndpoint = sourceCluster?.endpoint @@ -270,7 +270,7 @@ export class StackComposer { } const targetClusterAuth = targetCluster?.auth - const targetVersion = this.getEngineVersion(targetCluster?.version || engineVersion) + const targetVersion = this.getEngineVersion(targetCluster?.version ?? engineVersion) const requiredFields: { [key: string]: any; } = {"stage":stage} for (let key in requiredFields) { @@ -284,12 +284,12 @@ export class StackComposer { if (stage.length > 15) { throw new Error(`Maximum allowed stage name length is 15 characters but received ${stage}`) } - const clusterDomainName = domainName ? domainName : `os-cluster-${stage}` + const clusterDomainName = domainName ?? `os-cluster-${stage}` let preexistingOrContainerTargetEndpoint if (targetCluster && osContainerServiceEnabled) { throw new Error("The following options are mutually exclusive as only one target cluster can be specified for a given deployment: [targetCluster, osContainerServiceEnabled]") } else if (targetCluster || osContainerServiceEnabled) { - preexistingOrContainerTargetEndpoint = targetCluster?.endpoint || "https://opensearch:9200" + preexistingOrContainerTargetEndpoint = targetCluster?.endpoint ?? "https://opensearch:9200" } const fargateCpuArch = validateFargateCpuArch(defaultFargateCpuArch) @@ -316,14 +316,14 @@ export class StackComposer { trafficReplayerCustomUserAgent = `${props.migrationsUserAgent};${trafficReplayerUserAgentSuffix}` } else { - trafficReplayerCustomUserAgent = trafficReplayerUserAgentSuffix ? trafficReplayerUserAgentSuffix : props.migrationsUserAgent + trafficReplayerCustomUserAgent = trafficReplayerUserAgentSuffix ?? props.migrationsUserAgent } if (sourceClusterDisabled && (sourceCluster || captureProxyESServiceEnabled || elasticsearchServiceEnabled || captureProxyServiceEnabled)) { throw new Error("A source cluster must be specified by one of: [sourceCluster, captureProxyESServiceEnabled, elasticsearchServiceEnabled, captureProxyServiceEnabled]"); } - const deployId = addOnMigrationDeployId ? addOnMigrationDeployId : defaultDeployId + const deployId = addOnMigrationDeployId ?? defaultDeployId // If enabled re-use existing VPC and/or associated resources or create new let networkStack: NetworkStack|undefined @@ -404,10 +404,8 @@ export class StackComposer { this.addDependentStacks(openSearchStack, [networkStack]) this.stacks.push(openSearchStack) servicesYaml.target_cluster = openSearchStack.targetClusterYaml; - } else { - if (targetCluster) { - servicesYaml.target_cluster = targetCluster - } + } else if (targetCluster) { + servicesYaml.target_cluster = targetCluster } let migrationStack @@ -448,7 +446,7 @@ export class StackComposer { this.addDependentStacks(osContainerStack, [migrationStack]) this.stacks.push(osContainerStack) servicesYaml.target_cluster = new ClusterYaml({ - endpoint: preexistingOrContainerTargetEndpoint || "", + endpoint: preexistingOrContainerTargetEndpoint ?? "", auth: new ClusterAuth({noAuth: new ClusterNoAuth()}) }) } diff --git a/deployment/cdk/opensearch-service-migration/test/createApp.test.ts b/deployment/cdk/opensearch-service-migration/test/createApp.test.ts new file mode 100644 index 000000000..9d8906f71 --- /dev/null +++ b/deployment/cdk/opensearch-service-migration/test/createApp.test.ts @@ -0,0 +1,77 @@ +import { App, Tags } from 'aws-cdk-lib'; +import { createApp } from '../bin/createApp'; +import { StackComposer } from '../lib/stack-composer'; + +jest.mock('fs', () => ({ + readFileSync: jest.fn().mockReturnValue('1.0.0\n'), +})); + +jest.mock('aws-cdk-lib', () => ({ + App: jest.fn().mockImplementation(() => ({ + node: { + tryGetContext: jest.fn(), + }, + })), + Tags: { + of: jest.fn().mockReturnValue({ + add: jest.fn(), + }), + }, + Stack: class MockStack {}, +})); + +jest.mock('../lib/stack-composer'); + +describe('createApp', () => { + const originalEnv = process.env; + + beforeEach(() => { + jest.clearAllMocks(); + process.env = { ...originalEnv }; + }); + + afterAll(() => { + process.env = originalEnv; + }); + + it('should create an App instance with correct configuration', () => { + // Set up environment variables + process.env.CDK_DEFAULT_ACCOUNT = 'test-account'; + process.env.CDK_DEFAULT_REGION = 'test-region'; + process.env.MIGRATIONS_APP_REGISTRY_ARN = 'test-arn'; + process.env.CUSTOM_REPLAYER_USER_AGENT = 'test-user-agent'; + + const consoleSpy = jest.spyOn(console, 'info').mockImplementation(); + const mockAddTag = jest.fn(); + Tags.of = jest.fn().mockReturnValue({ add: mockAddTag }); + + const app = createApp(); + + // Verify App creation + expect(App).toHaveBeenCalled(); + + // Verify tag addition + expect(mockAddTag).toHaveBeenCalledWith('migration_deployment', '1.0.0'); + + // Verify StackComposer creation + expect(StackComposer).toHaveBeenCalledWith( + expect.any(Object), + { + migrationsAppRegistryARN: 'test-arn', + customReplayerUserAgent: 'test-user-agent', + migrationsSolutionVersion: '1.0.0', + env: { account: 'test-account', region: 'test-region' }, + } + ); + + // Verify console log + expect(consoleSpy).toHaveBeenCalledWith( + expect.stringContaining('App Registry mode is enabled for CFN stack tracking') + ); + + // Verify app is returned + expect(app).toBeDefined(); + + consoleSpy.mockRestore(); + }); +}); \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/test/lambda/acm-cert-importer-handler.test.ts b/deployment/cdk/opensearch-service-migration/test/lambda/acm-cert-importer-handler.test.ts new file mode 100644 index 000000000..1e1baaca3 --- /dev/null +++ b/deployment/cdk/opensearch-service-migration/test/lambda/acm-cert-importer-handler.test.ts @@ -0,0 +1,177 @@ +import { handler } from '../../lib/lambda/acm-cert-importer-handler'; +import { CloudFormationCustomResourceCreateEvent, CloudFormationCustomResourceUpdateEvent, CloudFormationCustomResourceDeleteEvent, Context } from 'aws-lambda'; +import { ACMClient, ImportCertificateCommand, DeleteCertificateCommand } from '@aws-sdk/client-acm'; +import * as forge from 'node-forge'; +import * as https from 'https'; + +jest.mock('@aws-sdk/client-acm'); +jest.mock('node-forge'); +jest.mock('https'); + +describe('ACM Certificate Importer Handler', () => { + let mockContext: Context; + + beforeEach(() => { + mockContext = { + callbackWaitsForEmptyEventLoop: false, + functionName: 'mockFunctionName', + functionVersion: 'mockFunctionVersion', + invokedFunctionArn: 'mockInvokedFunctionArn', + memoryLimitInMB: '128', + awsRequestId: 'mockAwsRequestId', + logGroupName: 'mockLogGroupName', + logStreamName: 'mockLogStreamName', + getRemainingTimeInMillis: jest.fn(), + done: jest.fn(), + fail: jest.fn(), + succeed: jest.fn() + }; + + process.env.AWS_REGION = 'us-west-2'; + + (https.request as jest.Mock).mockImplementation((options, callback) => { + const mockResponse = { + statusCode: 200, + statusMessage: 'OK', + }; + callback(mockResponse); + return { + on: jest.fn(), + write: jest.fn(), + end: jest.fn(), + }; + }); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + test('Create: should generate and import a self-signed certificate', async () => { + const mockEvent: CloudFormationCustomResourceCreateEvent = { + RequestType: 'Create', + ServiceToken: 'mockServiceToken', + ResponseURL: 'https://mockurl.com', + StackId: 'mockStackId', + RequestId: 'mockRequestId', + LogicalResourceId: 'mockLogicalResourceId', + ResourceType: 'Custom::ACMCertificateImporter', + ResourceProperties: { + ServiceToken: 'mockServiceToken' + } + }; + + const mockCertificate = 'mockCertificate'; + const mockPrivateKey = 'mockPrivateKey'; + const mockCertificateChain = 'mockCertificateChain'; + const mockCertificateArn = 'arn:aws:acm:us-west-2:123456789012:certificate/mock-certificate-id'; + + (forge.pki.rsa.generateKeyPair as jest.Mock).mockReturnValue({ + publicKey: 'mockPublicKey', + privateKey: 'mockPrivateKey' + }); + + const mockCert = { + publicKey: 'mockPublicKey', + serialNumber: '01', + validity: { + notBefore: new Date(), + notAfter: new Date() + }, + setSubject: jest.fn(), + setIssuer: jest.fn(), + setExtensions: jest.fn(), + sign: jest.fn() + }; + (forge.pki.createCertificate as jest.Mock).mockReturnValue(mockCert); + + (forge.pki.certificateToPem as jest.Mock).mockReturnValue(mockCertificate); + (forge.pki.privateKeyToPem as jest.Mock).mockReturnValue(mockPrivateKey); + + const mockSendFn = jest.fn().mockResolvedValue({ CertificateArn: mockCertificateArn }); + (ACMClient as jest.Mock).mockImplementation(() => ({ + send: mockSendFn + })); + + const result = await handler(mockEvent, mockContext); + + expect(result.Status).toBe('SUCCESS'); + expect(result.PhysicalResourceId).toBe(mockCertificateArn); + expect(result.Data).toEqual({ CertificateArn: mockCertificateArn }); + expect(mockSendFn).toHaveBeenCalledWith(expect.any(ImportCertificateCommand)); + + expect(forge.pki.rsa.generateKeyPair).toHaveBeenCalledWith(2048); + expect(mockCert.setSubject).toHaveBeenCalledWith([{ name: 'commonName', value: 'localhost' }]); + expect(mockCert.setIssuer).toHaveBeenCalledWith([{ name: 'commonName', value: 'localhost' }]); + expect(mockCert.setExtensions).toHaveBeenCalledWith(expect.arrayContaining([ + { name: 'basicConstraints', cA: true }, + { name: 'subjectAltName', altNames: [{ type: 2, value: 'localhost' }] }, + { name: 'keyUsage', keyCertSign: true, digitalSignature: true, keyEncipherment: true }, + { name: 'extKeyUsage', serverAuth: true, clientAuth: true } + ])); + expect(mockCert.sign).toHaveBeenCalledWith(expect.anything(), forge.md.sha1.create()); + + // Wait for any pending promises to resolve + await new Promise(process.nextTick); + }); + + test('Update: should return the existing physical resource id', async () => { + const mockEvent: CloudFormationCustomResourceUpdateEvent = { + RequestType: 'Update', + ServiceToken: 'mockServiceToken', + ResponseURL: 'https://mockurl.com', + StackId: 'mockStackId', + RequestId: 'mockRequestId', + LogicalResourceId: 'mockLogicalResourceId', + PhysicalResourceId: 'existingArn', + ResourceType: 'Custom::ACMCertificateImporter', + ResourceProperties: { + ServiceToken: 'mockServiceToken' + }, + OldResourceProperties: {} + }; + + const result = await handler(mockEvent, mockContext); + + expect(result.Status).toBe('SUCCESS'); + expect(result.PhysicalResourceId).toBe('existingArn'); + expect(result.Data).toEqual({}); + expect(ACMClient).not.toHaveBeenCalled(); + + // Wait for any pending promises to resolve + await new Promise(process.nextTick); + }); + + test('Delete: should delete the certificate', async () => { + const mockEvent: CloudFormationCustomResourceDeleteEvent = { + RequestType: 'Delete', + ServiceToken: 'mockServiceToken', + ResponseURL: 'https://mockurl.com', + StackId: 'mockStackId', + RequestId: 'mockRequestId', + LogicalResourceId: 'mockLogicalResourceId', + PhysicalResourceId: 'arnToDelete', + ResourceType: 'Custom::ACMCertificateImporter', + ResourceProperties: { + ServiceToken: 'mockServiceToken' + } + }; + + const mockSendFn = jest.fn().mockResolvedValue({}); + (ACMClient as jest.Mock).mockImplementation(() => ({ + send: mockSendFn + })); + + const result = await handler(mockEvent, mockContext); + + expect(result.Status).toBe('SUCCESS'); + expect(result.PhysicalResourceId).toBe('arnToDelete'); + expect(result.Data).toEqual({ CertificateArn: 'arnToDelete' }); + expect(mockSendFn).toHaveBeenCalledWith(expect.any(DeleteCertificateCommand)); + expect(mockSendFn).toHaveBeenCalledTimes(1); + expect(DeleteCertificateCommand).toHaveBeenCalledWith({ CertificateArn: 'arnToDelete' }); + + // Wait for any pending promises to resolve + await new Promise(process.nextTick); + }); +}); \ No newline at end of file