Merge remote-tracking branch 'origin/main' into user-agent-console

# Conflicts:
#	deployment/cdk/opensearch-service-migration/bin/app.ts
#	deployment/cdk/opensearch-service-migration/lib/stack-composer.ts

TrafficCapture/README.md

@@ -110,27 +110,46 @@ The body of the messages is sometimes gzipped which makes it difficult to repres
 and responses is base64 encoded before it is logged. This makes the files stable, but not human-readable.
 
 We have provided a utility script that can parse these files and output them to a human-readable format: the bodies are
-base64 decoded, un-gzipped if applicable, and parsed as JSON if applicable. They're then saved back to JSON format on disk.
+base64 decoded and parsed as JSON if applicable. They're then saved back to JSON format to stdout or file.
 
 To use this utility from the Migration Console,
 ```sh
-$ ./humanReadableLogs.py --help
-usage: humanReadableLogs.py [-h] [--outfile OUTFILE] infile
-
-positional arguments:
-  infile             Path to input logged tuple file.
-
-options:
-  -h, --help         show this help message and exit
-  --outfile OUTFILE  Path for output human readable tuple file.
-
-# By default, the output file is the same path as the input file, but the file name is prefixed with `readable-`.
-$ ./humanReadableLogs.py /shared_replayer_output/tuples.log
-Input file: /shared_replayer_output/tuples.log; Output file: /shared_replayer_output/readable-tuples.log
-
+$ console tuples show --help
+Usage: console tuples convert [OPTIONS]
+
+Options:
+  --in FILENAME
+  --out FILENAME
+  --help          Show this message and exit.
+
+# By default, the input and output files are `stdin` and `stdout` respectively, so they can be piped together with other tools.
+$ console tuples show --in /shared-logs-output/traffic-replayer-default/86ca83e66197/tuples/mini_tuples.log | jq
+{
+  "sourceRequest": {
+    "Request-URI": "/",
+    "Method": "GET",
+    "HTTP-Version": "HTTP/1.1",
+    "Host": "capture-proxy:9200",
+    "User-Agent": "python-requests/2.32.3",
+    "Accept-Encoding": "gzip, deflate, zstd",
+    "Accept": "*/*",
+    "Connection": "keep-alive",
+    "Authorization": "Basic YWRtaW46YWRtaW4=",
+    "body": ""
+  },
+  "sourceResponse": {
+    "HTTP-Version": {
+      "keepAliveDefault": true
+    },
+    "Status-Code": 200,
+    "Reason-Phrase": "OK",
+    ...
+  },
+  ...
+}
 # A specific output file can also be specified.
-$ ./humanReadableLogs.py /shared_replayer_output/tuples.log --outfile local-tuples.log
-Input file: /shared_replayer_output/tuples.log; Output file: local-tuples.log
+$ console tuples show --in /shared_replayer_output/tuples.log --out local-tuples.log
+Converted tuples output to local-tuples.log
 ```
 
 ### Capture Kafka Offloader

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile

@@ -27,9 +27,6 @@ COPY osiPipelineTemplate.yaml /root/
 COPY msk-iam-auth.properties /root/kafka-tools/aws
 COPY kafkaCmdRef.md /root/kafka-tools
 
-COPY humanReadableLogs.py /root/
-RUN chmod ug+x /root/humanReadableLogs.py
-
 COPY showFetchMigrationCommand.sh /root/
 RUN chmod ug+x /root/showFetchMigrationCommand.sh
 
@@ -57,7 +54,7 @@ RUN echo '. /.venv/bin/activate' >> /etc/profile.d/venv.sh
 RUN dnf install -y bash-completion
 RUN echo '. /etc/profile.d/bash_completion.sh' >> ~/.bashrc && \
     echo '. /etc/profile.d/venv.sh' >> ~/.bashrc && \
-    echo '@echo Welcome to the Migration Assistant Console' >> ~/.bashrc
+    echo 'echo Welcome to the Migration Assistant Console' >> ~/.bashrc
 # Set ENV to control startup script in /bin/sh mode
 ENV ENV=/root/.bashrc
 

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py

@@ -1,5 +1,6 @@
 import json
 from pprint import pprint
+import sys
 import click
 import console_link.middleware.clusters as clusters_
 import console_link.middleware.metrics as metrics_
@@ -8,6 +9,7 @@
 import console_link.middleware.metadata as metadata_
 import console_link.middleware.replay as replay_
 import console_link.middleware.kafka as kafka_
+import console_link.middleware.tuples as tuples_
 
 from console_link.models.utils import ExitCode
 from console_link.environment import Environment
@@ -485,6 +487,26 @@ def completion(ctx, config_file, json, shell):
         ctx.exit(1)
 
 
+@cli.group(name="tuples")
+@click.pass_obj
+def tuples_group(ctx):
+    """ All commands related to tuples. """
+    pass
+
+
+@tuples_group.command()
+@click.option('--in', 'inputfile',
+              type=click.File('r'),
+              default=sys.stdin)
+@click.option('--out', 'outputfile',
+              type=click.File('a'),
+              default=sys.stdout)
+def show(inputfile, outputfile):
+    tuples_.convert(inputfile, outputfile)
+    if outputfile != sys.stdout:
+        click.echo(f"Converted tuples output to {outputfile.name}")
+
+
 #################################################
 
 if __name__ == "__main__":

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/middleware/tuples.py

@@ -0,0 +1,10 @@
+import logging
+import typing
+
+from console_link.models.tuple_reader import TupleReader
+logger = logging.getLogger(__name__)
+
+
+def convert(inputfile: typing.TextIO, ouptutfile: typing.TextIO):
+    tuple_reader = TupleReader()
+    tuple_reader.transform_stream(inputfile, ouptutfile)