From b3e133c61c458048d4c5360364d441a611da9833 Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Tue, 12 Sep 2023 16:51:48 -0500 Subject: [PATCH] Draft lambda to export cloudwatch logs to s3 bucket; add aws-sdk cloudwatch-logs and ssm to package files. --- infrastructure/infrastructure.tf | 39 +++++- .../infrastructure_bucket_policy.tpl | 19 +++ infrastructure/lambdas/cloudwatch-to-s3.ts | 125 ++++++++++++++++++ 3 files changed, 177 insertions(+), 6 deletions(-) create mode 100644 infrastructure/infrastructure_bucket_policy.tpl create mode 100644 infrastructure/lambdas/cloudwatch-to-s3.ts diff --git a/infrastructure/infrastructure.tf b/infrastructure/infrastructure.tf index 83d6ffa5d..c7c84781e 100644 --- a/infrastructure/infrastructure.tf +++ b/infrastructure/infrastructure.tf @@ -1,17 +1,44 @@ -resource "aws_s3_bucket" "infrastructure" { - name = var.infrastructure_bucket_name +resource "aws_cloudwatch_log_group" "infrastructure" { + name = var.infrastructure_log_group_name + retention_in_days = 3653 + kms_key_id = aws_kms_key.key.arn tags = { Project = var.project Stage = var.stage } } -resource "aws_cloudwatch_log_group" "infrastructure" { - name = var.infrastructure_log_group_name - retention_in_days = 3653 - kms_key_id = aws_kms_key.key.arn +resource "aws_s3_bucket" "infrastructure_bucket" { + name = var.infrastructure_bucket_name tags = { Project = var.project Stage = var.stage } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "infrastructure_bucket" { + bucket = aws_s3_bucket.infrastructure_bucket.id + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_versioning" "infrastructure_bucket" { + bucket = aws_s3_bucket.infrastructure_bucket.id +} + +resource "aws_s3_bucket_logging" "infrastructure_bucket" { + bucket = aws_s3_bucket.infrastructure_bucket.id + target_bucket = aws_s3_bucket.logging_bucket.id + target_prefix = "infrastructure_bucket/" +} + +data "template_file" "infrastructure_bucket_policy" { + template = file("infrastructure_bucket_policy.tpl") + vars = { + accountId = data.aws_caller_identity.current.account_id + bucketName = var.infrastructure_bucket_name + } } \ No newline at end of file diff --git a/infrastructure/infrastructure_bucket_policy.tpl b/infrastructure/infrastructure_bucket_policy.tpl new file mode 100644 index 000000000..138bd7c6a --- /dev/null +++ b/infrastructure/infrastructure_bucket_policy.tpl @@ -0,0 +1,19 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "${accountId}" + }, + "Action": [ + "s3:Get*", + "s3:List*" + ], + "Resource": [ + "arn:aws:s3:::${bucketName}", + "arn:aws:s3:::${bucketName}/*" + ] + } + ] +} \ No newline at end of file diff --git a/infrastructure/lambdas/cloudwatch-to-s3.ts b/infrastructure/lambdas/cloudwatch-to-s3.ts new file mode 100644 index 000000000..d8b63b317 --- /dev/null +++ b/infrastructure/lambdas/cloudwatch-to-s3.ts @@ -0,0 +1,125 @@ +import { + CloudWatchLogsClient, + DescribeLogGroupsCommand, + DescribeLogGroupsRequest, + LogGroup, + ListTagsForResourceCommand, + CreateExportTaskCommand +} from '@aws-sdk/client-cloudwatch-logs'; +import { + SSMClient, + GetParameterCommand, + PutParameterCommand +} from '@aws-sdk/client-ssm'; + +const logs = new CloudWatchLogsClient({}); +const ssm = new SSMClient({}); +const region = process.env.AWS_REGION || 'us-east-1'; +const accountId = process.env.AWS_ACCOUNT_ID || '957221700844'; +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export const handler = async () => { + const extra_args: DescribeLogGroupsRequest = {}; + let log_groups: LogGroup[] = []; + const log_groups_to_export: string[] = []; + + if (!process.env.S3_BUCKET) { + console.error('Error: S3_BUCKET not defined'); + return; + } + + console.log('--> S3_BUCKET=' + process.env.S3_BUCKET); + + while (true) { + const response = await logs.send(new DescribeLogGroupsCommand(extra_args)); + log_groups = log_groups.concat(response.logGroups); + + if (!response.nextToken) { + break; + } + extra_args.nextToken = response.nextToken; + } + + for (const log_group of log_groups) { + const command = new ListTagsForResourceCommand({ + resourceArn: `arn:aws:logs:${region}:${accountId}:log-group:${log_group.logGroupName}` + }); + const response = await logs.send(command); + const log_group_tags = response.tags || {}; + + if (log_group_tags.ExportToS3 === 'true') { + log_groups_to_export.push(log_group.logGroupName); + } + await delay(10 * 1000); // prevents LimitExceededException (AWS allows only one export task at a time) + } + + for (const log_group_name of log_groups_to_export) { + const ssm_parameter_name = ( + '/log-exporter-last-export/' + log_group_name + ).replace('//', '/'); + let ssm_value = '0'; + + try { + const ssm_response = await ssm.send( + new GetParameterCommand({ Name: ssm_parameter_name }) + ); + ssm_value = ssm_response.Parameter?.Value || '0'; + } catch (error) { + if (error.name !== 'ParameterNotFound') { + console.error('Error fetching SSM parameter: ' + error.message); + } + } + + const export_to_time = Math.round(Date.now()); + + console.log( + '--> Exporting ' + log_group_name + ' to ' + process.env.S3_BUCKET + ); + + if (export_to_time - parseInt(ssm_value) < 24 * 60 * 60 * 1000) { + // Haven't been 24hrs from the last export of this log group + console.log(' Skipped until 24hrs from last export is completed'); + continue; + } + + try { + const response = await logs.send( + new CreateExportTaskCommand({ + logGroupName: log_group_name, + from: parseInt(ssm_value), + to: export_to_time, + destination: process.env.S3_BUCKET, + destinationPrefix: log_group_name + .replace(/^\//, '') + .replace(/\/$/, '') + }) + ); + + console.log(' Task created: ' + response.taskId); + await new Promise((resolve) => setTimeout(resolve, 5000)); + } catch (error) { + if (error.name === 'LimitExceededException') { + console.log( + ' Need to wait until all tasks are finished (LimitExceededException). Continuing later...' + ); + return; + } + console.error( + ' Error exporting ' + + log_group_name + + ': ' + + (error.message || JSON.stringify(error)) + ); + continue; + } + + await ssm.send( + new PutParameterCommand({ + Name: ssm_parameter_name, + Type: 'String', + Value: export_to_time.toString(), + Overwrite: true + }) + ); + } +};