diff --git a/backend/env.yml b/backend/env.yml index 301404984..4a922725a 100644 --- a/backend/env.yml +++ b/backend/env.yml @@ -41,6 +41,7 @@ staging: EXPORT_BUCKET_NAME: cisa-crossfeed-staging-exports PE_API_URL: ${ssm:/crossfeed/staging/PE_API_URL} REPORTS_BUCKET_NAME: cisa-crossfeed-staging-reports + CLOUDWATCH_BUCKET_NAME: cisa-crossfeed-staging-cloudwatch prod: DB_DIALECT: 'postgres' @@ -76,6 +77,7 @@ prod: EXPORT_BUCKET_NAME: cisa-crossfeed-prod-exports PE_API_URL: ${ssm:/crossfeed/staging/PE_API_URL} REPORTS_BUCKET_NAME: cisa-crossfeed-prod-reports + CLOUDWATCH_BUCKET_NAME: cisa-crossfeed-prod-cloudwatch dev-vpc: securityGroupIds: diff --git a/backend/package.json b/backend/package.json index 6b2c0ee97..ac6912bd8 100644 --- a/backend/package.json +++ b/backend/package.json @@ -7,6 +7,8 @@ }, "engineStrict": true, "dependencies": { + "@aws-sdk/client-cloudwatch-logs": "^3.417.0", + "@aws-sdk/client-ssm": "^3.414.0", "@elastic/elasticsearch": "~7.10.0", "@thefaultvault/tfv-cpe-parser": "^1.3.0", "aws-sdk": "^2.1352.0", @@ -105,4 +107,4 @@ }, "author": "", "license": "ISC" -} \ No newline at end of file +} diff --git a/backend/src/tasks/cloudwatchToS3.ts b/backend/src/tasks/cloudwatchToS3.ts new file mode 100644 index 000000000..7c106cb36 --- /dev/null +++ b/backend/src/tasks/cloudwatchToS3.ts @@ -0,0 +1,130 @@ +import { + CloudWatchLogsClient, + DescribeLogGroupsCommand, + DescribeLogGroupsRequest, + LogGroup, + ListTagsForResourceCommand, + CreateExportTaskCommand +} from '@aws-sdk/client-cloudwatch-logs'; +import { + SSMClient, + GetParameterCommand, + PutParameterCommand +} from '@aws-sdk/client-ssm'; + +const logs = new CloudWatchLogsClient({}); +const ssm = new SSMClient({}); +const region = process.env.AWS_REGION || 'us-east-1'; +const accountId = process.env.AWS_ACCOUNT_ID || '957221700844'; +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export const handler = async () => { + const extra_args: DescribeLogGroupsRequest = {}; + let log_groups: LogGroup[] = []; + const log_groups_to_export: string[] = []; + + if (!process.env.CLOUDWATCH_BUCKET_NAME) { + console.error('Error: CLOUDWATCH_BUCKET_NAME not defined'); + return; + } + + console.log( + '--> CLOUDWATCH_BUCKET_NAME=' + process.env.CLOUDWATCH_BUCKET_NAME + ); + + while (true) { + const response = await logs.send(new DescribeLogGroupsCommand(extra_args)); + log_groups = log_groups.concat(response.logGroups!); + + if (!response.nextToken) { + break; + } + extra_args.nextToken = response.nextToken; + } + + for (const log_group of log_groups) { + const command = new ListTagsForResourceCommand({ + resourceArn: `arn:aws:logs:${region}:${accountId}:log-group:${log_group.logGroupName}` + }); + const response = await logs.send(command); + const log_group_tags = response.tags || {}; + + if (log_group_tags.ExportToS3 === 'true') { + log_groups_to_export.push(log_group.logGroupName!); + } + await delay(10 * 1000); // prevents LimitExceededException (AWS allows only one export task at a time) + } + + for (const log_group_name of log_groups_to_export) { + const ssm_parameter_name = ( + '/log-exporter-last-export/' + log_group_name + ).replace('//', '/'); + let ssm_value = '0'; + + try { + const ssm_response = await ssm.send( + new GetParameterCommand({ Name: ssm_parameter_name }) + ); + ssm_value = ssm_response.Parameter?.Value || '0'; + } catch (error) { + if (error.name !== 'ParameterNotFound') { + console.error('Error fetching SSM parameter: ' + error.message); + } + } + + const export_to_time = Math.round(Date.now()); + + console.log( + '--> Exporting ' + + log_group_name + + ' to ' + + process.env.CLOUDWATCH_BUCKET_NAME + ); + + if (export_to_time - parseInt(ssm_value) < 24 * 60 * 60 * 1000) { + // Haven't been 24hrs from the last export of this log group + console.log(' Skipped until 24hrs from last export is completed'); + continue; + } + + try { + const response = await logs.send( + new CreateExportTaskCommand({ + logGroupName: log_group_name, + from: parseInt(ssm_value), + to: export_to_time, + destination: process.env.CLOUDWATCH_BUCKET_NAME, + destinationPrefix: log_group_name + .replace(/^\//, '') + .replace(/\/$/, '') + }) + ); + + console.log(' Task created: ' + response.taskId); + await new Promise((resolve) => setTimeout(resolve, 5000)); + } catch (error) { + if (error.name === 'LimitExceededException') { + console.log( + ' Need to wait until all tasks are finished (LimitExceededException). Continuing later...' + ); + return; + } + console.error( + ' Error exporting ' + + log_group_name + + ': ' + + (error.message || JSON.stringify(error)) + ); + continue; + } + + await ssm.send( + new PutParameterCommand({ + Name: ssm_parameter_name, + Type: 'String', + Value: export_to_time.toString(), + Overwrite: true + }) + ); + } +}; diff --git a/backend/src/tasks/functions.yml b/backend/src/tasks/functions.yml index a7da691b5..64c55d4a1 100644 --- a/backend/src/tasks/functions.yml +++ b/backend/src/tasks/functions.yml @@ -1,3 +1,11 @@ +cloudwatchToS3: + handler: src/tasks/cloudwatchToS3.handler + timeout: 900 + events: + - schedule: rate(4 hours) + reservedConcurrency: 1 + memorySize: 4096 + scheduler: handler: src/tasks/scheduler.handler timeout: 900 diff --git a/frontend/src/components/Header.tsx b/frontend/src/components/Header.tsx index badcb8f06..f9d87e58e 100644 --- a/frontend/src/components/Header.tsx +++ b/frontend/src/components/Header.tsx @@ -246,7 +246,14 @@ const HeaderNoCtx: React.FC = (props) => { users: ALL_USERS, exact: false }, - { title: 'Feeds', path: '/feeds', users: ALL_USERS, exact: false }, + + /* + Hiding Feeds page until finished + { title: 'Feeds', + path: '/feeds', + users: ALL_USERS, + exact: false + },*/ /* Hiding Reports page until finished diff --git a/frontend/src/components/__tests__/__snapshots__/header.spec.tsx.snap b/frontend/src/components/__tests__/__snapshots__/header.spec.tsx.snap index c888d9875..83e5f878c 100644 --- a/frontend/src/components/__tests__/__snapshots__/header.spec.tsx.snap +++ b/frontend/src/components/__tests__/__snapshots__/header.spec.tsx.snap @@ -49,17 +49,6 @@ exports[`Header component matches snapshot 1`] = ` Inventory -
- - Feeds - -