Updated Capture/Viewer Nodes to pull config files from Param Store (#79)

* Arkime Config Files uploaded to Param Store Signed-off-by: Chris Helma <[email protected]> * Arkime file Param Store paths piped to containers Signed-off-by: Chris Helma <[email protected]> * Updated Containers to pull config files from Param Store Signed-off-by: Chris Helma <[email protected]> * Updates per PR Discussion Signed-off-by: Chris Helma <[email protected]> --------- Signed-off-by: Chris Helma <[email protected]>
arkime · Jul 6, 2023 · a351752 · a351752
1 parent 5e555e1
commit a351752
Show file tree

Hide file tree

Showing 23 changed files with 528 additions and 57 deletions.
diff --git a/cdk-lib/capture-stacks/capture-bucket-stack.ts b/cdk-lib/capture-stacks/capture-bucket-stack.ts
@@ -4,7 +4,7 @@ import * as kms from 'aws-cdk-lib/aws-kms';
 import * as s3 from 'aws-cdk-lib/aws-s3';
 import * as ssm from 'aws-cdk-lib/aws-ssm';
 
-import * as plan from '../core/capacity-plan';
+import * as plan from '../core/context-types';
 import { ProductStack } from 'aws-cdk-lib/aws-servicecatalog';
 
 

diff --git a/cdk-lib/capture-stacks/capture-nodes-stack.ts b/cdk-lib/capture-stacks/capture-nodes-stack.ts
@@ -17,11 +17,12 @@ import * as path from 'path'
 import { Construct } from 'constructs';
 
 import * as constants from '../core/constants';
-import * as plan from '../core/capacity-plan';
+import * as plan from '../core/context-types';
 import {ClusterSsmValue} from '../core/ssm-wrangling';
-import * as user from '../core/user-config';
+import * as types from '../core/context-types';
 
 export interface CaptureNodesStackProps extends cdk.StackProps {
+    readonly arkimeFilesMap: types.ArkimeFilesMap;
     readonly captureBucket: s3.Bucket;
     readonly captureBucketKey: kms.Key;
     readonly captureVpc: ec2.Vpc;
@@ -30,7 +31,7 @@ export interface CaptureNodesStackProps extends cdk.StackProps {
     readonly osPassword: secretsmanager.Secret;
     readonly planCluster: plan.ClusterPlan;
     readonly ssmParamNameCluster: string;
-    readonly userConfig: user.UserConfig;
+    readonly userConfig: types.UserConfig;
 }
 
 export class CaptureNodesStack extends cdk.Stack {
@@ -141,6 +142,13 @@ export class CaptureNodesStack extends cdk.Stack {
                 resources: [ksmEncryptionKey.keyArn]
             }),
         );
+        taskDefinition.addToTaskRolePolicy(
+            new iam.PolicyStatement({
+                effect: iam.Effect.ALLOW,
+                actions: ['ssm:GetParameter'], // Container pulls configuration from Parameter Store
+                resources: [`arn:aws:ssm:${this.region}:${this.account}:parameter*`]
+            }),
+        );
         props.osPassword.grantRead(taskDefinition.taskRole);
         props.captureBucket.grantReadWrite(taskDefinition.taskRole);
         props.captureBucketKey.grantEncryptDecrypt(taskDefinition.taskRole);
@@ -157,6 +165,8 @@ export class CaptureNodesStack extends cdk.Stack {
             image: ecs.ContainerImage.fromAsset(path.resolve(__dirname, '..', '..', 'docker-capture-node')),
             logging: new ecs.AwsLogDriver({ streamPrefix: 'CaptureNodes', mode: ecs.AwsLogDriverMode.NON_BLOCKING }),
             environment: {
+                'ARKIME_CONFIG_INI_LOC': props.arkimeFilesMap.captureIniLoc,
+                'ARKIME_ADD_FILE_LOCS': JSON.stringify(props.arkimeFilesMap.captureAddFileLocs),
                 'AWS_REGION': this.region, // Seems not to be defined in this container, strangely
                 'BUCKET_NAME': props.captureBucket.bucketName,
                 'CLUSTER_NAME': props.clusterName,

diff --git a/cdk-lib/capture-stacks/capture-vpc-stack.ts b/cdk-lib/capture-stacks/capture-vpc-stack.ts
@@ -4,7 +4,7 @@ import * as ec2 from 'aws-cdk-lib/aws-ec2';
 import * as logs from 'aws-cdk-lib/aws-logs';
 import { Stack, StackProps } from 'aws-cdk-lib';
 
-import * as plan from '../core/capacity-plan';
+import * as plan from '../core/context-types';
 
 export interface CaptureVpcStackProps extends StackProps {
     readonly planCluster: plan.ClusterPlan;

diff --git a/cdk-lib/capture-stacks/opensearch-domain-stack.ts b/cdk-lib/capture-stacks/opensearch-domain-stack.ts
@@ -7,7 +7,7 @@ import {Domain, EngineVersion, TLSSecurityPolicy} from 'aws-cdk-lib/aws-opensear
 import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager';
 import * as ssm from 'aws-cdk-lib/aws-ssm';
 
-import * as plan from '../core/capacity-plan';
+import * as plan from '../core/context-types';
 
 
 export interface OpenSearchDomainStackProps extends StackProps {

diff --git a/cdk-lib/cloud-demo.ts b/cdk-lib/cloud-demo.ts
@@ -44,6 +44,7 @@ switch(params.type) {
 
         const captureNodesStack = new CaptureNodesStack(app, params.nameCaptureNodesStack, {
             env: env,
+            arkimeFilesMap: params.arkimeFileMap,
             captureBucket: captureBucketStack.bucket,
             captureBucketKey: captureBucketStack.bucketKey,
             captureVpc: captureVpcStack.vpc,
@@ -60,6 +61,7 @@ switch(params.type) {
 
         const viewerNodesStack = new ViewerNodesStack(app, params.nameViewerNodesStack, {
             env: env,
+            arkimeFilesMap: params.arkimeFileMap,
             arnViewerCert: params.nameViewerCertArn,
             captureBucket: captureBucketStack.bucket,
             viewerVpc: captureVpcStack.vpc,

diff --git a/cdk-lib/core/command-params.ts b/cdk-lib/core/command-params.ts
@@ -1,5 +1,4 @@
-import * as plan from './capacity-plan';
-import * as user from './user-config';
+import * as types from './context-types';
 
 /**
  * Base type for receiving arguments from the Python side of the app.  These directly match the interface on the Python
@@ -13,6 +12,7 @@ export interface CommandParamsRaw { }
  */
 export interface ClusterMgmtParamsRaw extends CommandParamsRaw {
     type: 'ClusterMgmtParamsRaw';
+    arkimeFileMap: string;
     nameCluster: string;
     nameCaptureBucketStack: string;
     nameCaptureBucketSsmParam: string;
@@ -78,6 +78,7 @@ export interface DestroyDemoTrafficParams extends CommandParams {
  */
 export interface ClusterMgmtParams extends CommandParams {
     type: 'ClusterMgmtParams'
+    arkimeFileMap: types.ArkimeFilesMap;
     nameCluster: string;
     nameCaptureBucketStack: string;
     nameCaptureBucketSsmParam: string;
@@ -91,8 +92,8 @@ export interface ClusterMgmtParams extends CommandParams {
     nameViewerPassSsmParam: string;
     nameViewerUserSsmParam: string;
     nameViewerNodesStack: string;
-    planCluster: plan.ClusterPlan;
-    userConfig: user.UserConfig;
+    planCluster: types.ClusterPlan;
+    userConfig: types.UserConfig;
 }
 
 /**

diff --git a/cdk-lib/core/capacity-plan.ts → cdk-lib/core/context-types.ts b/cdk-lib/core/capacity-plan.ts → cdk-lib/core/context-types.ts
@@ -66,3 +66,24 @@ export interface ClusterPlan {
     osDomain: OSDomainPlan;
     s3: S3Plan;
 }
+
+/**
+ * Structure to hold the user's input configuration
+ */
+export interface UserConfig {
+    expectedTraffic: number;
+    spiDays: number;
+    historyDays: number;
+    replicas: number;
+    pcapDays: number;
+}
+
+/**
+ * Structure to hold the mapping of Arkime files to their path in the data store
+ */
+export interface ArkimeFilesMap {
+    captureIniLoc: string;
+    captureAddFileLocs: string[];
+    viewerIniLoc: string;
+    viewerAddFileLocs: string[];
+}
diff --git a/cdk-lib/core/context-wrangling.ts b/cdk-lib/core/context-wrangling.ts
@@ -1,5 +1,4 @@
 import * as cdk from 'aws-cdk-lib';
-import * as plan from './capacity-plan';
 import * as prms from './command-params';
 import {CDK_CONTEXT_CMD_VAR, CDK_CONTEXT_REGION_VAR, CDK_CONTEXT_PARAMS_VAR, ManagementCmd} from './constants';
 
@@ -86,6 +85,7 @@ function validateArgs(args: ValidateArgs) : (prms.ClusterMgmtParams | prms.Deplo
                 type: 'ClusterMgmtParams',
                 awsAccount: args.awsAccount,
                 awsRegion: args.awsRegion,
+                arkimeFileMap: JSON.parse(rawClusterMgmtParamsObj.arkimeFileMap),
                 nameCluster: rawClusterMgmtParamsObj.nameCluster,
                 nameCaptureBucketStack: rawClusterMgmtParamsObj.nameCaptureBucketStack,
                 nameCaptureBucketSsmParam: rawClusterMgmtParamsObj.nameCaptureBucketSsmParam,

diff --git a/cdk-lib/core/ssm-wrangling.ts b/cdk-lib/core/ssm-wrangling.ts
@@ -1,4 +1,4 @@
-import * as plan from '../core/capacity-plan';
+import * as plan from './context-types';
 import * as user from '../core/user-config';
 
 /**

diff --git a/cdk-lib/core/user-config.ts b/cdk-lib/core/user-config.ts
diff --git a/cdk-lib/viewer-stacks/viewer-nodes-stack.ts b/cdk-lib/viewer-stacks/viewer-nodes-stack.ts
@@ -11,8 +11,10 @@ import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager';
 import * as ssm from 'aws-cdk-lib/aws-ssm';
 import * as path from 'path'
 import { Construct } from 'constructs';
+import * as types from '../core/context-types';
 
 export interface ViewerNodesStackProps extends cdk.StackProps {
+    readonly arkimeFilesMap: types.ArkimeFilesMap;
     readonly arnViewerCert: string;
     readonly captureBucket: s3.Bucket;
     readonly viewerVpc: ec2.Vpc;
@@ -62,6 +64,13 @@ export class ViewerNodesStack extends cdk.Stack {
                 resources: [ksmEncryptionKey.keyArn]
             }),
         );
+        taskDefinition.addToTaskRolePolicy(
+            new iam.PolicyStatement({
+                effect: iam.Effect.ALLOW,
+                actions: ['ssm:GetParameter'], // Container pulls configuration from Parameter Store
+                resources: [`arn:aws:ssm:${this.region}:${this.account}:parameter*`]
+            }),
+        );
         props.osPassword.grantRead(taskDefinition.taskRole);
         props.captureBucket.grantRead(taskDefinition.taskRole);
         props.osDomain.grantReadWrite(taskDefinition.taskRole);
@@ -73,6 +82,8 @@ export class ViewerNodesStack extends cdk.Stack {
             image: ecs.ContainerImage.fromAsset(path.resolve(__dirname, '..', '..', 'docker-viewer-node')),
             logging: new ecs.AwsLogDriver({ streamPrefix: 'ViewerNodes', mode: ecs.AwsLogDriverMode.NON_BLOCKING }),
             environment: {
+                'ARKIME_CONFIG_INI_LOC': props.arkimeFilesMap.viewerIniLoc,
+                'ARKIME_ADD_FILE_LOCS': JSON.stringify(props.arkimeFilesMap.viewerAddFileLocs),
                 'AWS_REGION': this.region, // Seems not to be defined in this container, strangely
                 'BUCKET_NAME': props.captureBucket.bucketName,
                 'CLUSTER_NAME': props.clusterName,

diff --git a/docker-capture-node/Dockerfile b/docker-capture-node/Dockerfile
@@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND noninteractive
 
 # Get required and quality-of-life utilities
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends wget curl ca-certificates unzip less mandoc man-db vim libmagic1 iproute2 dnsutils net-tools tcpdump
+    apt-get install -y --no-install-recommends wget curl ca-certificates unzip less mandoc man-db vim libmagic1 iproute2 dnsutils net-tools tcpdump jq
 
 # Install the AWS CLI so we can pull passwords from AWS Secrets Manager at runtime
 RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
@@ -20,8 +20,5 @@ RUN wget -O arkime.deb https://s3.amazonaws.com/files.molo.ch/arkime-main_ubuntu
 # Download the rir & oui files
 RUN /opt/arkime/bin/arkime_update_geo.sh
 
-COPY ./arkime_config.ini /opt/arkime/etc/config.ini
-COPY ./default.rules /opt/arkime/etc/default.rules
-
 COPY ./run_capture_node.sh /run_capture_node.sh
 CMD ["/run_capture_node.sh"]
diff --git a/docker-capture-node/run_capture_node.sh b/docker-capture-node/run_capture_node.sh
@@ -5,6 +5,8 @@ set -e
 echo "============================================================"
 echo "Cluster: $CLUSTER_NAME"
 echo "Role: Capture Node"
+echo "Arkime Config INI Datastore Location: $ARKIME_CONFIG_INI_LOC"
+echo "Arkime Additional File Datastore Locations: $ARKIME_ADD_FILE_LOCS"
 echo "AWS Region: $AWS_REGION"
 echo "Bucket Name: $BUCKET_NAME"
 echo "LB Healthcheck Port: $LB_HEALTH_PORT"
@@ -13,22 +15,52 @@ echo "OpenSearch Secret Arn: $OPENSEARCH_SECRET_ARN"
 echo "S3 Storage Class: $S3_STORAGE_CLASS"
 echo "============================================================"
 
+# Pull our configuration files from the cloud
+function write_file_from_datastore() {
+    datastore_location=$1
+
+    # Retrieve our file from the cloud and account for wacky escaping
+    param_val=$(aws ssm get-parameter --name "$datastore_location" --query Parameter.Value)
+    corrected_string=$(echo "$param_val" | sed 's/\\\"/\"/g' | sed 's/\\\\/\\/g') # Remove extra escaping
+    corrected_string=$(echo "$corrected_string" | sed 's/^"//' | sed 's/"$//') # Remove starting/ending quotes
+
+    # Pull out the values we need
+    system_path=$(echo "$corrected_string" | jq -r '.system_path')
+    echo "System Path: $system_path" >&2
+    contents=$(echo "$corrected_string" | jq -r '.contents')
+
+    # Write the file to disk
+    echo -e "$contents" > "$system_path"
+
+    # Return the path to the calling context
+    echo "$system_path"
+}
+
+echo "$ARKIME_ADD_FILE_LOCS" | jq -r '.[]' | while IFS= read -r path; do
+    echo "Processing File in Datastore: $path"
+    full_file_path=$(write_file_from_datastore "$path")
+    echo "Written to: $full_file_path"
+done
+
+echo "Processing config.ini in Datastore: $ARKIME_CONFIG_INI_LOC"
+config_ini_path=$(write_file_from_datastore "$ARKIME_CONFIG_INI_LOC")
+echo "Written to: $config_ini_path"
+
 # Pull configuration from ENV and AWS in order to set up our Arkime install.  The ENV variables come from the Fargate
 # Container definition.  We perform some escaping of the our replacement strings for safety.
 # See: https://stackoverflow.com/questions/407523/escape-a-string-for-a-sed-replace-pattern
-echo "Configuring /opt/arkime/etc/config.ini ..."
+echo "Configuring $config_ini_path ..."
 ESCAPED_ENDPOINT=$(printf '%s\n' "$OPENSEARCH_ENDPOINT" | sed -e 's/[\/&]/\\&/g')
-sed -i'' "s/_ENDPOINT_/$ESCAPED_ENDPOINT/g" /opt/arkime/etc/config.ini
+sed -i'' "s/_OS_ENDPOINT_/$ESCAPED_ENDPOINT/g" "$config_ini_path"
 
 OPENSEARCH_PASS=$(aws secretsmanager get-secret-value --secret-id $OPENSEARCH_SECRET_ARN --output text --query SecretString)
 BASE64_AUTH=$(echo -n "admin:$OPENSEARCH_PASS" | base64)
-sed -i'' "s/_AUTH_/$BASE64_AUTH/g" /opt/arkime/etc/config.ini
+sed -i'' "s/_OS_AUTH_/$BASE64_AUTH/g" "$config_ini_path"
 
-sed -i'' "s/_BUCKET_/$BUCKET_NAME/g" /opt/arkime/etc/config.ini
-sed -i'' "s/_HEALTH_PORT_/$LB_HEALTH_PORT/g" /opt/arkime/etc/config.ini
-sed -i'' "s/_REGION_/$AWS_REGION/g" /opt/arkime/etc/config.ini
-sed -i'' "s/_STORAGE_CLASS_/$S3_STORAGE_CLASS/g" /opt/arkime/etc/config.ini
-echo "Successfully configured /opt/arkime/etc/config.ini"
+sed -i'' "s/_PCAP_BUCKET_/$BUCKET_NAME/g" "$config_ini_path"
+sed -i'' "s/_HEALTH_PORT_/$LB_HEALTH_PORT/g" "$config_ini_path"
+sed -i'' "s/_AWS_REGION_/$AWS_REGION/g" "$config_ini_path"
+echo "Successfully configured $config_ini_path"
 
 echo "Testing connection/creds to OpenSearch domain $OPENSEARCH_ENDPOINT ..."
 curl -u admin:$OPENSEARCH_PASS -X GET https://$OPENSEARCH_ENDPOINT:443
@@ -46,4 +78,4 @@ chown nobody /opt/arkime/raw # Unneeded when using S3 offload
 
 # Start Arkime Capture
 echo "Running Arkime Capture process ..."
-/opt/arkime/bin/capture
+/opt/arkime/bin/capture --config "$config_ini_path"
diff --git a/docker-viewer-node/Dockerfile b/docker-viewer-node/Dockerfile
@@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND noninteractive
 
 # Get required and quality-of-life utilities
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends wget curl ca-certificates unzip less mandoc man-db vim libmagic1 iproute2 dnsutils net-tools tcpdump
+    apt-get install -y --no-install-recommends wget curl ca-certificates unzip less mandoc man-db vim libmagic1 iproute2 dnsutils net-tools tcpdump jq
 
 # Install the AWS CLI so we can pull the OpenSearch Domain password from AWS Secrets Manager at runtime
 RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
@@ -16,7 +16,6 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2
 RUN wget -O arkime.deb https://s3.amazonaws.com/files.molo.ch/arkime-main_ubuntu20_amd64.deb && \
     apt-get install -y ./arkime.deb && \
     rm -rf ./arkime.deb
-COPY ./arkime_config.ini /opt/arkime/etc/config.ini
 
 COPY ./run_viewer_node.sh /run_viewer_node.sh
 CMD ["/run_viewer_node.sh"]