Merge pull request opensearch-project#1050 from peternied/aws-features

Add a script to determine the AWS resources used

deployment/cdk/opensearch-service-migration/.gitignore

@@ -9,3 +9,4 @@ dist
 .cdk.staging
 cdk.out
 certs
+cdk-synth-output

deployment/cdk/opensearch-service-migration/README.md

@@ -73,12 +73,12 @@ A set of demo context values (using the `demo-deploy` label) has been set in the
 
 This demo solution can be deployed with the following command:
 ```shell
-cdk deploy "*" --c contextId=demo-deploy --require-approval never --concurrency 3
+./deploy.sh demo-deploy
 ```
 
 Additionally, another context block in the `cdk.context.json` could be created with say the label `uat-deploy` with its own custom context configuration and be deployed with the command:
 ```shell
-cdk deploy "*" --c contextId=uat-deploy --require-approval never --concurrency 3
+./deploy.sh uat-deploy
 ```
 **Note**: Separate deployments within the same account and region should use unique `stage` context values to avoid resource naming conflicts when deploying (**Except** in the multiple replay scenario stated [here](#how-to-run-multiple-traffic-replayer-scenarios))
 
@@ -155,7 +155,7 @@ To give an example of this process, a user could decide to configure an addition
 ```
 And then deploy this additional infrastructure with the command:
 ```shell
-cdk deploy "*" --c contextId=demo-addon1 --require-approval never --concurrency 3
+./deploy.sh demo-addon1
 ```
 
 Finally, the additional infrastructure can be removed with:

deployment/cdk/opensearch-service-migration/awsFeatureUsage.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+set -e
+
+if [ -z "$1" ]; then
+    echo "Error: contextId is required. Please pass it as the first argument to the script."
+    echo "Usage: $0 <contextId>"
+    exit 1
+fi
+
+contextId="$1"
+
+output_dir="cdk-synth-output"
+rm -rf "$output_dir"
+mkdir -p "$output_dir"
+
+echo "Synthesizing all stacks..."
+raw_stacks=$(cdk list --ci --context contextId=$contextId)
+
+echo "$raw_stacks" | sed -E 's/ *\(.*\)//' | while read -r stack; do
+    echo "Synthesizing stack: $stack"
+    cdk synth $stack --ci --context contextId=$contextId > "$output_dir/$stack.yaml"
+done
+
+echo "Finding resource usage from synthesized stacks..."
+echo "-----------------------------------"
+echo "IAM Policy Actions:"
+
+grep -h -A 1000 "PolicyDocument:" "$output_dir"/*.yaml | \
+grep -E "Action:" -A 50 | \
+grep -E "^[ \t-]+[a-zA-Z0-9]+:[a-zA-Z0-9\*]+" | \
+grep -vE "^[ \t-]+(aws:|arn:)" | \
+sed -E 's/^[ \t-]+//' | \
+sort -u
+
+echo "-----------------------------------"
+echo "Resources Types:"
+
+grep -h -E " Type: AWS" "$output_dir"/*.yaml | \
+sed -E 's/^[ \t]*Type: //' | \
+sort -u

deployment/cdk/opensearch-service-migration/deploy.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+if [ -z "$1" ]; then
+    echo "Error: contextId is required. Please pass it as the first argument to the script."
+    echo "Usage: $0 <contextId>"
+    exit 1
+fi
+
+contextId="$1"
+
+export CDK_CLI_COMMAND=deploy
+cdk deploy "*" --c contextId=$contextId --require-approval never --concurrency 5

deployment/cdk/opensearch-service-migration/lib/cdk-logger.ts

@@ -0,0 +1,20 @@
+/**
+ * Logger that prints messages in the CDK ecosystem
+ */
+export class CdkLogger {
+    private constructor() {}
+
+    static info(message: string) {
+      if (process.env.CDK_CLI_COMMAND === 'deploy') {
+        console.log(message);
+      }
+    }
+
+    static warn(message: string) {
+      console.log(message);
+    }
+
+    static error(message: string) {
+      console.log(message);
+    }
+  }

deployment/cdk/opensearch-service-migration/lib/index.ts

@@ -1,3 +1,4 @@
+export * from './cdk-logger';
 export * from './common-utilities';
 export * from './lambda'
 export * from './migration-assistance-stack';

deployment/cdk/opensearch-service-migration/lib/network-stack.ts

@@ -16,6 +16,7 @@ import { Stack } from "aws-cdk-lib";
 import { createMigrationStringParameter, getMigrationStringParameterName, isStackInGovCloud, MigrationSSMParameter } from "./common-utilities";
 import { StringParameter } from "aws-cdk-lib/aws-ssm";
 import { GatewayVpcEndpoint, InterfaceVpcEndpoint } from "aws-cdk-lib/aws-ec2";
+import { CdkLogger } from "./cdk-logger";
 
 export interface NetworkStackProps extends StackPropsExt {
     readonly vpcId?: string;
@@ -75,7 +76,7 @@ export class NetworkStack extends Stack {
                 onePerAz: true
             }).subnetIds
         }
-        console.info(`Detected VPC with ${vpc.privateSubnets.length} private subnets, ${vpc.publicSubnets.length} public subnets, and ${vpc.isolatedSubnets.length} isolated subnets`)
+        CdkLogger.info(`Detected VPC with ${vpc.privateSubnets.length} private subnets, ${vpc.publicSubnets.length} public subnets, and ${vpc.isolatedSubnets.length} isolated subnets`)
         if (uniqueAzPrivateSubnets.length < 2) {
             throw new Error(`Not enough AZs (${uniqueAzPrivateSubnets.length} unique AZs detected) used for private subnets to meet 2 or 3 AZ requirement`)
         }

deployment/cdk/opensearch-service-migration/lib/opensearch-domain-stack.ts

@@ -23,6 +23,7 @@ import {
   createMigrationStringParameter,
   getMigrationStringParameterValue
 } from "./common-utilities";
+import { CdkLogger } from "./cdk-logger";
 
 
 export interface OpensearchDomainStackProps extends StackPropsExt {
@@ -109,7 +110,7 @@ export class OpenSearchDomainStack extends Stack {
       stage,
     });
     if (domain.masterUserPassword && !adminUserSecret) {
-      console.log(`An OpenSearch domain fine-grained access control user was configured without an existing Secrets Manager secret, will not create SSM Parameter: /migration/${stage}/${deployId}/osUserAndSecret`)
+      CdkLogger.info(`An OpenSearch domain fine-grained access control user was configured without an existing Secrets Manager secret, will not create SSM Parameter: /migration/${stage}/${deployId}/osUserAndSecret`)
     } else if (domain.masterUserPassword && adminUserSecret) {
       createMigrationStringParameter(this, `${adminUserName} ${adminUserSecret.secretArn}`, {
           parameter: MigrationSSMParameter.OS_USER_AND_SECRET_ARN,

deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts

@@ -19,6 +19,7 @@ import {OtelCollectorSidecar} from "./migration-otel-collector-sidecar";
 import { ECSReplayerYaml } from "../migration-services-yaml";
 import { SharedLogFileSystem } from "../components/shared-log-file-system";
 import {Secret} from "aws-cdk-lib/aws-secretsmanager";
+import { CdkLogger } from "../cdk-logger";
 
 
 export interface TrafficReplayerProps extends StackPropsExt {
@@ -92,7 +93,7 @@ export class TrafficReplayerStack extends MigrationServiceCore {
         if (props.clusterAuthDetails.basicAuth) {
             let secret;
             if (props.clusterAuthDetails.basicAuth.password) {
-                console.warn("Password passed in plain text, this is insecure and will leave" +
+                CdkLogger.warn("Password passed in plain text, this is insecure and will leave" +
                     "your password exposed.")
                 secret = new Secret(this,"ReplayerClusterPasswordSecret", {
                     secretName: `replayer-user-secret-${props.stage}-${deployId}`,

deployment/cdk/opensearch-service-migration/lib/stack-composer.ts

@@ -25,6 +25,7 @@ import {
 } from "./common-utilities";
 import {ReindexFromSnapshotStack} from "./service-stacks/reindex-from-snapshot-stack";
 import {ClientOptions, ClusterYaml, ServicesYaml} from "./migration-services-yaml";
+import { CdkLogger } from "./cdk-logger";
 
 export interface StackPropsExt extends StackProps {
     readonly stage: string,
@@ -70,7 +71,7 @@ export class StackComposer {
                     return JSON.parse(option)
                 } catch (e) {
                     if (e instanceof SyntaxError) {
-                        console.error(`Unable to parse option: ${optionName} with expected type: ${expectedType}`)
+                        CdkLogger.error(`Unable to parse option: ${optionName} with expected type: ${expectedType}`)
                     }
                     throw e
                 }
@@ -144,9 +145,7 @@ export class StackComposer {
             throw new Error("Required context field 'contextId' not provided")
         }
         const contextJSON = this.parseContextBlock(scope, contextId)
-        console.log('Received following context block for deployment: ')
-        console.log(contextJSON)
-        console.log('End of context block.')
+        CdkLogger.info(`Context block for '${contextId}':\n---\n${JSON.stringify(contextJSON, null, 3)}\n---`);
 
         const stage = this.getContextForType('stage', 'string', defaultValues, contextJSON)
 
@@ -221,8 +220,8 @@ export class StackComposer {
         let sourceClusterDefinition = this.getContextForType('sourceCluster', 'object', defaultValues, contextJSON)
 
         if (!sourceClusterDefinition && (sourceClusterEndpointField || sourceClusterDisabledField)) {
-            console.warn("`sourceClusterDisabled` and `sourceClusterEndpoint` are being deprecated in favor of a `sourceCluster` object.")
-            console.warn("Please update your CDK context block to use the `sourceCluster` object.")
+            CdkLogger.warn("`sourceClusterDisabled` and `sourceClusterEndpoint` are being deprecated in favor of a `sourceCluster` object.")
+            CdkLogger.warn("Please update your CDK context block to use the `sourceCluster` object.")
             sourceClusterDefinition = {
                 "disabled": sourceClusterDisabledField,
                 "endpoint": sourceClusterEndpointField,
@@ -242,11 +241,11 @@ export class StackComposer {
         let targetClusterDefinition = this.getContextForType('targetCluster', 'object', defaultValues, contextJSON)
         const usePreexistingTargetCluster = !!(targetClusterEndpointField || targetClusterDefinition)
         if (!targetClusterDefinition && usePreexistingTargetCluster) {
-            console.warn("`targetClusterEndpoint` is being deprecated in favor of a `targetCluster` object.")
-            console.warn("Please update your CDK context block to use the `targetCluster` object.")
+            CdkLogger.warn("`targetClusterEndpoint` is being deprecated in favor of a `targetCluster` object.")
+            CdkLogger.warn("Please update your CDK context block to use the `targetCluster` object.")
             let auth: any = {"type": "none"}
             if (fineGrainedManagerUserName || fineGrainedManagerUserSecretManagerKeyARN) {
-                console.warn(`Use of ${fineGrainedManagerUserName} and ${fineGrainedManagerUserSecretManagerKeyARN} with a preexisting target cluster
+                CdkLogger.warn(`Use of ${fineGrainedManagerUserName} and ${fineGrainedManagerUserSecretManagerKeyARN} with a preexisting target cluster
                     will be deprecated in favor of using a \`targetCluster\` object. Please update your CDK context block.`)
                 auth = {
                     "type": "basic",
@@ -267,7 +266,7 @@ export class StackComposer {
         // Ensure that target version is not defined in multiple places, but `engineVersion` is set as a default value, so this is
         // a warning instead of an error.
         if (usePreexistingTargetCluster && engineVersion) {
-            console.warn("The `engineVersion` value will be ignored because it's only used when a domain is being provisioned by this tooling" +
+            CdkLogger.warn("The `engineVersion` value will be ignored because it's only used when a domain is being provisioned by this tooling" +
                 "and in this case, `targetCluster` was provided to define an existing target cluster."
             )
         }
@@ -283,7 +282,7 @@ export class StackComposer {
             }
         }
         if (addOnMigrationDeployId && vpcId) {
-            console.warn("Add-on deployments will use the original deployment 'vpcId' regardless of passed 'vpcId' values")
+            CdkLogger.warn("Add-on deployments will use the original deployment 'vpcId' regardless of passed 'vpcId' values")
         }
         if (stage.length > 15) {
             throw new Error(`Maximum allowed stage name length is 15 characters but received ${stage}`)
@@ -302,7 +301,6 @@ export class StackComposer {
         if (captureProxyServiceEnabled || captureProxyESServiceEnabled || trafficReplayerServiceEnabled || kafkaBrokerServiceEnabled) {
             streamingSourceType = determineStreamingSourceType(kafkaBrokerServiceEnabled)
         } else {
-            console.log("MSK is not enabled and will not be deployed.")
             streamingSourceType = StreamingSourceType.DISABLED
         }