Skip to content

Commit

Permalink
Fix cloudwatchToS3 Lambda (#2302)
Browse files Browse the repository at this point in the history 
* Refactor lambda to export all logs (not just those with ExportToS3 tag); refactor variable names for consistency with project.

* Remove console logs used for troubleshooting.
  • Loading branch information
@Matthew-Grayson
Matthew-Grayson authored Oct 12, 2023
1 parent 0a5b72e commit d8741fe
Showing 1 changed file with 35 additions and 67 deletions.
102 changes: 35 additions & 67 deletions backend/src/tasks/cloudwatchToS3.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import {
DescribeLogGroupsCommand,
DescribeLogGroupsRequest,
LogGroup,
ListTagsForResourceCommand,
CreateExportTaskCommand
} from '@aws-sdk/client-cloudwatch-logs';
import {
Expand All @@ -14,113 +13,81 @@ import {

const logs = new CloudWatchLogsClient({});
const ssm = new SSMClient({});
const region = process.env.AWS_REGION || 'us-east-1';
const accountId = process.env.AWS_ACCOUNT_ID || '957221700844';
const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));

export const handler = async () => {
const extra_args: DescribeLogGroupsRequest = {};
let log_groups: LogGroup[] = [];
const log_groups_to_export: string[] = [];
const args: DescribeLogGroupsRequest = {};
let logGroups: LogGroup[] = [];
const logBucketName = process.env.CLOUDWATCH_BUCKET_NAME;

if (!process.env.CLOUDWATCH_BUCKET_NAME) {
console.error('Error: CLOUDWATCH_BUCKET_NAME not defined');
if (!logBucketName) {
console.error('Error: logBucketName not defined');
return;
}

console.log(
'--> CLOUDWATCH_BUCKET_NAME=' + process.env.CLOUDWATCH_BUCKET_NAME
);
console.log('--> logBucketName=' + logBucketName);

while (true) {
const response = await logs.send(new DescribeLogGroupsCommand(extra_args));
console.log(`response: ${JSON.stringify(response)}`);
log_groups = log_groups.concat(response.logGroups!);
console.log(`log_groups: ${JSON.stringify(log_groups)}`);
const response = await logs.send(new DescribeLogGroupsCommand(args));
logGroups = logGroups.concat(response.logGroups!);
if (!response.nextToken) {
break;
}
extra_args.nextToken = response.nextToken;
args.nextToken = response.nextToken;
}

for (const log_group of log_groups) {
const command = new ListTagsForResourceCommand({
resourceArn: `arn:aws:logs:${region}:${accountId}:log-group:${log_group.logGroupName}`
});
console.log(`Processing log group: ${log_group.logGroupName}`);
console.log(`command: ${JSON.stringify(command)}`);
const response = await logs.send(command);
console.log(`log group response: ${JSON.stringify(response)}`);
const log_group_tags = response.tags || {};

if (log_group_tags.ExportToS3 === 'true') {
log_groups_to_export.push(log_group.logGroupName!);
}
console.log(
`log_groups_to_export: ${JSON.stringify(log_groups_to_export)}`
);
await delay(10 * 1000); // prevents LimitExceededException (AWS allows only one export task at a time)
}

for (const log_group_name of log_groups_to_export) {
console.log('Processing log group: ' + log_group_name);
const ssm_parameter_name = (
'/log-exporter-last-export/' + log_group_name
for (const logGroup of logGroups) {
const logGroupName = logGroup.logGroupName!;
console.log('Processing log group: ' + logGroupName);
const ssmParameterName = (
'/log-exporter-last-export/' + logGroupName
).replace('//', '/');
let ssm_value = '0';
let ssmValue = '0';

try {
const ssm_response = await ssm.send(
new GetParameterCommand({ Name: ssm_parameter_name })
const ssmResponse = await ssm.send(
new GetParameterCommand({ Name: ssmParameterName })
);
ssm_value = ssm_response.Parameter?.Value || '0';
ssmValue = ssmResponse.Parameter?.Value || '0';
} catch (error) {
if (error.name !== 'ParameterNotFound') {
console.error('Error fetching SSM parameter: ' + error.message);
}
console.error(`error: ${error.message}`);
}

const export_to_time = Math.round(Date.now());
const exportTime = Math.round(Date.now());

console.log(
'--> Exporting ' +
log_group_name +
' to ' +
process.env.CLOUDWATCH_BUCKET_NAME
);
console.log('--> Exporting ' + logGroupName + ' to ' + logBucketName);

if (export_to_time - parseInt(ssm_value) < 24 * 60 * 60 * 1000) {
// Haven't been 24hrs from the last export of this log group
console.log(' Skipped until 24hrs from last export is completed');
if (exportTime - parseInt(ssmValue) < 24 * 60 * 60 * 1000) {
console.log(
'Skipped: log group was already exported in the last 24 hours'
);
continue;
}

try {
const response = await logs.send(
new CreateExportTaskCommand({
logGroupName: log_group_name,
from: parseInt(ssm_value),
to: export_to_time,
destination: process.env.CLOUDWATCH_BUCKET_NAME,
destinationPrefix: log_group_name
.replace(/^\//, '')
.replace(/\/$/, '')
logGroupName: logGroupName,
from: parseInt(ssmValue),
to: exportTime,
destination: logBucketName,
destinationPrefix: logGroupName.replace(/^\//, '').replace(/\/$/, '')
})
);

console.log(' Task created: ' + response.taskId);
await new Promise((resolve) => setTimeout(resolve, 5000));
} catch (error) {
if (error.name === 'LimitExceededException') {
console.log(
' Need to wait until all tasks are finished (LimitExceededException). Continuing later...'
);
console.log(error.message);
return;
}
console.error(
' Error exporting ' +
log_group_name +
'Error exporting ' +
logGroupName +
': ' +
(error.message || JSON.stringify(error))
);
Expand All @@ -129,11 +96,12 @@ export const handler = async () => {

await ssm.send(
new PutParameterCommand({
Name: ssm_parameter_name,
Name: ssmParameterName,
Type: 'String',
Value: export_to_time.toString(),
Value: exportTime.toString(),
Overwrite: true
})
);
}
await delay(10 * 1000); // prevents LimitExceededException (AWS allows only one export task at a time)
};

0 comments on commit d8741fe

Please sign in to comment.