From 453677cecbb56f74c647c66bc355cc8d2fdaada0 Mon Sep 17 00:00:00 2001 From: "Grayson, Matthew" Date: Wed, 11 Oct 2023 15:46:50 -0500 Subject: [PATCH] Refactor lambda to export all logs (not just those with ExportToS3 tag); refactor variable names for consistency with project. --- backend/src/tasks/cloudwatchToS3.ts | 100 ++++++++++------------------ 1 file changed, 34 insertions(+), 66 deletions(-) diff --git a/backend/src/tasks/cloudwatchToS3.ts b/backend/src/tasks/cloudwatchToS3.ts index 7364cecb2..d495a708f 100644 --- a/backend/src/tasks/cloudwatchToS3.ts +++ b/backend/src/tasks/cloudwatchToS3.ts @@ -3,7 +3,6 @@ import { DescribeLogGroupsCommand, DescribeLogGroupsRequest, LogGroup, - ListTagsForResourceCommand, CreateExportTaskCommand } from '@aws-sdk/client-cloudwatch-logs'; import { @@ -14,66 +13,44 @@ import { const logs = new CloudWatchLogsClient({}); const ssm = new SSMClient({}); -const region = process.env.AWS_REGION || 'us-east-1'; -const accountId = process.env.AWS_ACCOUNT_ID || '957221700844'; const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); export const handler = async () => { - const extra_args: DescribeLogGroupsRequest = {}; - let log_groups: LogGroup[] = []; - const log_groups_to_export: string[] = []; + const args: DescribeLogGroupsRequest = {}; + let logGroups: LogGroup[] = []; + const logBucketName = process.env.CLOUDWATCH_BUCKET_NAME; - if (!process.env.CLOUDWATCH_BUCKET_NAME) { - console.error('Error: CLOUDWATCH_BUCKET_NAME not defined'); + if (!logBucketName) { + console.error('Error: logBucketName not defined'); return; } - console.log( - '--> CLOUDWATCH_BUCKET_NAME=' + process.env.CLOUDWATCH_BUCKET_NAME - ); + console.log('--> logBucketName=' + logBucketName); while (true) { - const response = await logs.send(new DescribeLogGroupsCommand(extra_args)); + const response = await logs.send(new DescribeLogGroupsCommand(args)); console.log(`response: ${JSON.stringify(response)}`); - log_groups = log_groups.concat(response.logGroups!); - console.log(`log_groups: ${JSON.stringify(log_groups)}`); + logGroups = logGroups.concat(response.logGroups!); + console.log(`logGroups: ${JSON.stringify(logGroups)}`); if (!response.nextToken) { break; } - extra_args.nextToken = response.nextToken; + args.nextToken = response.nextToken; } - for (const log_group of log_groups) { - const command = new ListTagsForResourceCommand({ - resourceArn: `arn:aws:logs:${region}:${accountId}:log-group:${log_group.logGroupName}` - }); - console.log(`Processing log group: ${log_group.logGroupName}`); - console.log(`command: ${JSON.stringify(command)}`); - const response = await logs.send(command); - console.log(`log group response: ${JSON.stringify(response)}`); - const log_group_tags = response.tags || {}; - - if (log_group_tags.ExportToS3 === 'true') { - log_groups_to_export.push(log_group.logGroupName!); - } - console.log( - `log_groups_to_export: ${JSON.stringify(log_groups_to_export)}` - ); - await delay(10 * 1000); // prevents LimitExceededException (AWS allows only one export task at a time) - } - - for (const log_group_name of log_groups_to_export) { - console.log('Processing log group: ' + log_group_name); - const ssm_parameter_name = ( - '/log-exporter-last-export/' + log_group_name + for (const logGroup of logGroups) { + const logGroupName = logGroup.logGroupName!; + console.log('Processing log group: ' + logGroupName); + const ssmParameterName = ( + '/log-exporter-last-export/' + logGroupName ).replace('//', '/'); - let ssm_value = '0'; + let ssmValue = '0'; try { - const ssm_response = await ssm.send( - new GetParameterCommand({ Name: ssm_parameter_name }) + const ssmResponse = await ssm.send( + new GetParameterCommand({ Name: ssmParameterName }) ); - ssm_value = ssm_response.Parameter?.Value || '0'; + ssmValue = ssmResponse.Parameter?.Value || '0'; } catch (error) { if (error.name !== 'ParameterNotFound') { console.error('Error fetching SSM parameter: ' + error.message); @@ -81,31 +58,23 @@ export const handler = async () => { console.error(`error: ${error.message}`); } - const export_to_time = Math.round(Date.now()); + const exportTime = Math.round(Date.now()); - console.log( - '--> Exporting ' + - log_group_name + - ' to ' + - process.env.CLOUDWATCH_BUCKET_NAME - ); + console.log('--> Exporting ' + logGroupName + ' to ' + logBucketName); - if (export_to_time - parseInt(ssm_value) < 24 * 60 * 60 * 1000) { - // Haven't been 24hrs from the last export of this log group - console.log(' Skipped until 24hrs from last export is completed'); + if (exportTime - parseInt(ssmValue) < 24 * 60 * 60 * 1000) { + console.log('Skipped until 24hrs from last export'); continue; } try { const response = await logs.send( new CreateExportTaskCommand({ - logGroupName: log_group_name, - from: parseInt(ssm_value), - to: export_to_time, - destination: process.env.CLOUDWATCH_BUCKET_NAME, - destinationPrefix: log_group_name - .replace(/^\//, '') - .replace(/\/$/, '') + logGroupName: logGroupName, + from: parseInt(ssmValue), + to: exportTime, + destination: logBucketName, + destinationPrefix: logGroupName.replace(/^\//, '').replace(/\/$/, '') }) ); @@ -113,14 +82,12 @@ export const handler = async () => { await new Promise((resolve) => setTimeout(resolve, 5000)); } catch (error) { if (error.name === 'LimitExceededException') { - console.log( - ' Need to wait until all tasks are finished (LimitExceededException). Continuing later...' - ); + console.log(error.message); return; } console.error( - ' Error exporting ' + - log_group_name + + 'Error exporting ' + + logGroupName + ': ' + (error.message || JSON.stringify(error)) ); @@ -129,11 +96,12 @@ export const handler = async () => { await ssm.send( new PutParameterCommand({ - Name: ssm_parameter_name, + Name: ssmParameterName, Type: 'String', - Value: export_to_time.toString(), + Value: exportTime.toString(), Overwrite: true }) ); } + await delay(10 * 1000); // prevents LimitExceededException (AWS allows only one export task at a time) };