From 93eb520640ea829f82f631d5900389bdf63654fd Mon Sep 17 00:00:00 2001 From: Suzanna-Volkov Date: Tue, 12 Jan 2021 17:41:44 +0100 Subject: [PATCH] Created files for easier testing --- .bumpversion.cfg | 8 + cloudformation-no-trail.json | 2 +- cloudformation-test.json | 572 +++++++++++++++++++++++++++++++++++ deploy-using-profile.sh | 16 + 4 files changed, 597 insertions(+), 1 deletion(-) create mode 100644 cloudformation-test.json create mode 100755 deploy-using-profile.sh diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 00766bd..1f24db4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -15,3 +15,11 @@ replace = v{new_version} [bumpversion:file:cloudformation.json] search = v{current_version} replace = v{new_version} + +[bumpversion:file:cloudformation-no-trail.json] +search = v{current_version} +replace = v{new_version} + +[bumpversion:file:cloudformation-test.json] +search = v{current_version} +replace = v{new_version} diff --git a/cloudformation-no-trail.json b/cloudformation-no-trail.json index 47707c3..08bc522 100644 --- a/cloudformation-no-trail.json +++ b/cloudformation-no-trail.json @@ -71,7 +71,7 @@ "Version" : { "Type" : "String", "Description" : "The version of the integration you want installed.", - "Default" : "v1.2.0" + "Default" : "v1.2.1" } }, "Conditions" : { diff --git a/cloudformation-test.json b/cloudformation-test.json new file mode 100644 index 0000000..af6c00a --- /dev/null +++ b/cloudformation-test.json @@ -0,0 +1,572 @@ +{ + "AWSTemplateFormatVersion" : "2010-09-09", + "Description" : "CloudWatch to Humio Integration for sending CloudWatch Logs and Metrics to Humio.", + "Parameters" : { + "HumioProtocol" : { + "Type" : "String", + "Description" : "The transport protocol used for delivering log/metric events to Humio. HTTPS is default and recommended.", + "Default" : "https" + }, + "HumioHost" : { + "Type" : "String", + "Description" : "The host to ship Humio log/metric events to.", + "Default" : "cloud.humio.com" + }, + "HumioIngestToken" : { + "Type" : "String", + "Description" : "The value of the ingest token for the repository from your Humio account to ship log/metric events to.", + "Default" : "", + "NoEcho" : true + }, + "HumioLambdaLogRetention" : { + "Type" : "Number", + "Description" : "Number of days to retain CloudWatch logs from the Humio Lambda functions.", + "Default": 1 + }, + "EnableCloudWatchLogsAutoSubscription" : { + "Type" : "String", + "AllowedValues" : [ + "true", + "false" + ], + "Description" : "Make the log ingester automatically subscribe to new log groups specified with the logs subscription prefix parameter. Set to 'true' to enable.", + "Default" : "true" + }, + "HumioCloudWatchLogsSubscriptionPrefix" : { + "Type" : "String", + "Description" : "Humio will only subscribe to log groups with the prefix specified.", + "Default" : "" + }, + "EnableCloudWatchLogsBackfillerAutoRun" : { + "Type" : "String", + "AllowedValues" : [ + "true", + "false" + ], + "Description" : "Make the backfiller run automatically when created. Set to 'true' to enable.", + "Default" : "true" + }, + "EnableVPCForIngesterLambdas" : { + "Type" : "String", + "AllowedValues" : [ + "true", + "false" + ], + "Description" : "Use a VPC for the lambda ingester functions. Set to 'true' to enable.", + "Default" : "false" + }, + "SecurityGroupIds" : { + "Type" : "CommaDelimitedList", + "Description" : "A comma separated list of security group ids for the VPC configuration regarding the ingester lambda functions. Only required if VPC is enabled." + }, + "SubnetIds" : { + "Type" : "CommaDelimitedList", + "Description" : "A comma separated list of subnet ids used by the VPC configuration that the ingester lamda functions will be deployed into. Only required if VPC is enabled." + }, + "HumioLambdaLogLevel" : { + "Type" : "String", + "AllowedValues" : ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + "Default" : "INFO" + }, + "Version" : { + "Type" : "String", + "Description" : "The version of the integration you want installed.", + "Default" : "v1.2.1" + } + }, + "Conditions" : { + "CreateAutoSubscriptionResources" : { + "Fn::Equals" : [ { "Ref" : "EnableCloudWatchLogsAutoSubscription" }, "true" ] + }, + "CreateHumioBackfillerAutoRunner" : { + "Fn::Equals" : [ { "Ref" : "EnableCloudWatchLogsBackfillerAutoRun" }, "true" ] + }, + "ConfigureVPCForIngesterLambdas" : { + "Fn::Equals" : [ { "Ref" : "EnableVPCForIngesterLambdas" }, "true" ] + } + }, + "Resources" : { + "HumioCloudWatchRole" : { + "Type" : "AWS::IAM::Role", + "Properties" : { + "AssumeRolePolicyDocument" : { + "Version" : "2012-10-17", + "Statement" : [ + { + "Action" : "sts:AssumeRole", + "Principal" : { + "Service" : [ + "lambda.amazonaws.com", + "apigateway.amazonaws.com", + "logs.amazonaws.com" + ] + }, + "Effect" : "Allow", + "Sid" : "" + } + ] + }, + "Policies" : [ + { + "PolicyName" : "humio_cloudwatch_role", + "PolicyDocument" : { + "Fn::If" : [ "ConfigureVPCForIngesterLambdas", + { + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Action" : [ + "lambda:GetFunction", + "lambda:InvokeFunction", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:DescribeSubscriptionFilters", + "logs:PutSubscriptionFilter", + "logs:DeleteSubscriptionFilter", + "logs:PutLogEvents", + "logs:GetLogEvents", + "logs:FilterLogEvents", + "cloudwatch:GetMetricData", + "cloudwatch:GetMetricStatistics", + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:CreateNetworkInterfacePermission", + "ec2:DeleteNetworkInterface" + ], + "Resource" : "*" + } + ] + }, + { + "Version" : "2012-10-17", + "Statement" : [ + { + "Effect" : "Allow", + "Action" : [ + "lambda:GetFunction", + "lambda:InvokeFunction", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:DescribeSubscriptionFilters", + "logs:PutSubscriptionFilter", + "logs:DeleteSubscriptionFilter", + "logs:PutLogEvents", + "logs:GetLogEvents", + "logs:FilterLogEvents", + "cloudwatch:GetMetricData", + "cloudwatch:GetMetricStatistics" + ], + "Resource" : "*" + } + ] + } + ] + } + } + ] + } + }, + "HumioCloudWatchLogsIngester" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Lambda::Function", + "Properties" : { + "Code" : { + "S3Bucket" : "cloudwatch2humio", + "S3Key" : { + "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] + } + }, + "Environment" : { + "Variables" : { + "humio_protocol" : { "Ref" : "HumioProtocol" }, + "humio_host" : { "Ref" : "HumioHost" }, + "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, + "log_level" : { "Ref" : "HumioLambdaLogLevel" } + } + }, + "VpcConfig" : { + "Fn::If" : [ "ConfigureVPCForIngesterLambdas", + { + "SecurityGroupIds" : { "Ref" : "SecurityGroupIds" }, + "SubnetIds" : { "Ref" : "SubnetIds" } + }, + { + "Ref" : "AWS::NoValue" + } + ] + }, + "Description" : "CloudWatch Logs to Humio ingester.", + "Handler" : "logs_ingester.lambda_handler", + "MemorySize" : "128", + "Role" : { + "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] + }, + "Runtime" : "python3.8", + "Timeout" : "300" + } + }, + "HumioCloudWatchLogsIngesterPermission" : { + "Type" : "AWS::Lambda::Permission", + "Properties" : { + "Action" : "lambda:InvokeFunction", + "FunctionName" : { + "Fn::GetAtt" : [ "HumioCloudWatchLogsIngester", "Arn" ] + }, + "Principal" : "logs.amazonaws.com" + } + }, + "HumioCloudWatchLogsIngesterLogGroup" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Logs::LogGroup", + "Properties" : { + "LogGroupName" : { + "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchLogsIngester" } ] ] + }, + "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } + } + }, + "HumioCloudWatchLogsSubscriber" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Lambda::Function", + "Properties" : { + "Code" : { + "S3Bucket" : "cloudwatch2humio", + "S3Key" : { + "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] + } + }, + "Environment" : { + "Variables" : { + "humio_log_ingester_arn" : { + "Fn::GetAtt" : [ "HumioCloudWatchLogsIngester", "Arn" ] + }, + "humio_subscription_prefix" : { "Ref" : "HumioCloudWatchLogsSubscriptionPrefix" }, + "log_level" : { "Ref" : "HumioLambdaLogLevel" } + } + }, + "Description" : "CloudWatch Logs to Humio log group subscriber.", + "Handler" : "logs_subscriber.lambda_handler", + "MemorySize" : "128", + "Role" : { + "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] + }, + "Runtime" : "python3.8", + "Timeout" : "300" + } + }, + "HumioCloudWatchLogsSubscriberPermission" : { + "Condition" : "CreateAutoSubscriptionResources", + "Type" : "AWS::Lambda::Permission", + "Properties" : { + "Action" : "lambda:InvokeFunction", + "FunctionName" : { + "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriber", "Arn" ] + }, + "Principal" : "events.amazonaws.com", + "SourceAccount" : { "Ref" : "AWS::AccountId" } + } + }, + "HumioCloudWatchLogsSubscriberPermission2" : { + "Condition" : "CreateAutoSubscriptionResources", + "Type" : "AWS::Lambda::Permission", + "Properties" : { + "Action" : "lambda:InvokeFunction", + "FunctionName" : { + "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriber", "Arn" ] + }, + "Principal" : "events.amazonaws.com", + "SourceArn" : { + "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriberEventRule", "Arn" ] + } + } + }, + "HumioCloudWatchLogsSubscriberLogGroup" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Logs::LogGroup", + "Properties" : { + "LogGroupName" : { + "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchLogsSubscriber" } ] ] + }, + "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } + } + }, + "HumioCloudWatchLogsBackfiller" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Lambda::Function", + "Properties" : { + "Code" : { + "S3Bucket" : "cloudwatch2humio", + "S3Key" : { + "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] + } + }, + "Environment" : { + "Variables" : { + "humio_log_ingester_arn" : { + "Fn::GetAtt" : [ "HumioCloudWatchLogsIngester", "Arn" ] + }, + "humio_subscription_prefix" : { "Ref" : "HumioCloudWatchLogsSubscriptionPrefix" }, + "humio_protocol" : { "Ref" : "HumioProtocol" }, + "humio_host" : { "Ref" : "HumioHost" }, + "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, + "log_level" : { "Ref" : "HumioLambdaLogLevel" } + } + }, + "Description" : "CloudWatch Logs to Humio logs backfiller.", + "Handler" : "logs_backfiller.lambda_handler", + "MemorySize" : "128", + "Role" : { + "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] + }, + "Runtime" : "python3.8", + "Timeout" : "300" + } + }, + "HumioCloudWatchLogsBackfillerPermission" : { + "Type" : "AWS::Lambda::Permission", + "Properties" : { + "Action" : "lambda:InvokeFunction", + "FunctionName" : { + "Fn::GetAtt" : [ "HumioCloudWatchLogsBackfiller", "Arn" ] + }, + "Principal" : "logs.amazonaws.com" + } + }, + "HumioCloudWatchLogsBackfillerLogGroup" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Logs::LogGroup", + "Properties" : { + "LogGroupName" : { + "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchLogsBackfiller" } ] ] + }, + "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } + } + }, + "HumioBackfillerAutoRunner" : { + "Condition" : "CreateHumioBackfillerAutoRunner", + "DependsOn" : [ "HumioCloudWatchLogsBackfiller" ], + "Type" : "Custom::BackfillerAutoRunner", + "Properties" : { + "ServiceToken" : { "Fn::GetAtt" : [ "HumioCloudWatchLogsBackfiller", "Arn" ] }, + "StackName" : { "Ref" : "AWS::StackName" } + } + }, + "HumioCloudWatchLogsSubscriberS3Bucket" : { + "Condition" : "CreateAutoSubscriptionResources", + "Type" : "AWS::S3::Bucket", + "Properties" : { + "AccessControl" : "BucketOwnerFullControl", + "BucketName" : { + "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio", "cloudtrail" ] ] + } + } + }, + "HumioCloudWatchLogsSubscriberS3BucketPolicy" : { + "Condition" : "CreateAutoSubscriptionResources", + "DependsOn" : [ "HumioCloudWatchLogsSubscriberS3Bucket" ], + "Type" : "AWS::S3::BucketPolicy", + "Properties" : { + "Bucket" : { "Ref" : "HumioCloudWatchLogsSubscriberS3Bucket" }, + "PolicyDocument" : { + "Version" : "2012-10-17", + "Statement" : [ + { + "Sid" : "AWSCloudTrailAclCheck20150319", + "Effect" : "Allow", + "Principal" : { + "Service" : "cloudtrail.amazonaws.com" + }, + "Action" : "s3:GetBucketAcl", + "Resource" : { + "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriberS3Bucket", "Arn" ] + } + }, + { + "Sid" : "AWSCloudTrailWrite20150319", + "Effect" : "Allow", + "Principal" : { + "Service" : "cloudtrail.amazonaws.com" + }, + "Action" : "s3:PutObject", + "Resource" : { + "Fn::Join" : [ "", [ { "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriberS3Bucket", "Arn" ] }, "/AWSLogs/", { "Ref" : "AWS::AccountId" }, "/*" ] ] + }, + "Condition" : { + "StringEquals" : { "s3:x-amz-acl" : "bucket-owner-full-control" } + } + } + ] + } + } + }, + "HumioCloudWatchLogsSubscriberCloudTrail" : { + "Condition" : "CreateAutoSubscriptionResources", + "DependsOn" : [ "HumioCloudWatchLogsSubscriberS3BucketPolicy" ], + "Type" : "AWS::CloudTrail::Trail", + "Properties" : { + "EnableLogFileValidation" : false, + "IncludeGlobalServiceEvents" : true, + "IsMultiRegionTrail" : true, + "IsLogging" : true, + "S3BucketName" : { + "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio", "cloudtrail" ] ] + }, + "TrailName" : { + "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio", { "Ref" : "AWS::AccountId" } ] ] + } + } + }, + "HumioCloudWatchLogsSubscriberEventRule" : { + "Condition" : "CreateAutoSubscriptionResources", + "DependsOn" : [ "HumioCloudWatchLogsSubscriber" ], + "Type" : "AWS::Events::Rule", + "Properties" : { + "Description" : "Humio log group auto subscription event rule.", + "EventPattern" : { + "source" : [ "aws.logs" ], + "detail-type" : [ "AWS API Call via CloudTrail" ], + "detail" : { + "eventSource" : [ "logs.amazonaws.com" ], + "eventName" : [ "CreateLogGroup" ] + } + }, + "Name" : { + "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio-auto-subscription-rule" ] ] + }, + "Targets" : [ + { + "Id" : { + "Fn::Join" : [ "-", [ { "Ref" : "AWS::StackName" }, "humio-auto-subscription-rule" ] ] + }, + "Arn" : { + "Fn::GetAtt" : [ "HumioCloudWatchLogsSubscriber", "Arn" ] + } + } + ] + } + }, + "HumioCloudWatchMetricIngester" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Lambda::Function", + "Properties" : { + "Code" : { + "S3Bucket" : "cloudwatch2humio", + "S3Key" : { + "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] + } + }, + "Environment" : { + "Variables" : { + "humio_protocol" : { "Ref" : "HumioProtocol" }, + "humio_host" : { "Ref" : "HumioHost" }, + "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, + "log_level" : { "Ref" : "HumioLambdaLogLevel" } + } + }, + "VpcConfig" : { + "Fn::If" : [ "ConfigureVPCForIngesterLambdas", + { + "SecurityGroupIds" : { "Ref" : "SecurityGroupIds" }, + "SubnetIds" : { "Ref" : "SubnetIds" } + }, + { + "Ref" : "AWS::NoValue" + } + ] + }, + "Description" : "CloudWatch Metrics to Humio ingester.", + "Handler" : "metric_ingester.lambda_handler", + "MemorySize" : "128", + "Role" : { + "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] + }, + "Runtime" : "python3.8", + "Timeout" : "300" + } + }, + "HumioCloudWatchMetricIngesterPermission" : { + "Type" : "AWS::Lambda::Permission", + "Properties" : { + "Action" : "lambda:InvokeFunction", + "FunctionName" : { + "Fn::GetAtt" : [ "HumioCloudWatchMetricIngester", "Arn" ] + }, + "Principal" : "logs.amazonaws.com" + } + }, + "HumioCloudWatchMetricIngesterLogGroup" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Logs::LogGroup", + "Properties" : { + "LogGroupName" : { + "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchMetricIngester" } ] ] + }, + "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } + } + }, + "HumioCloudWatchMetricStatisticsIngester" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Lambda::Function", + "Properties" : { + "Code" : { + "S3Bucket" : "cloudwatch2humio", + "S3Key" : { + "Fn::Join" : [ "_", [ { "Ref" : "Version" }, "cloudwatch2humio.zip" ] ] + } + }, + "Environment" : { + "Variables" : { + "humio_protocol" : { "Ref" : "HumioProtocol" }, + "humio_host" : { "Ref" : "HumioHost" }, + "humio_ingest_token" : { "Ref" : "HumioIngestToken" }, + "log_level" : { "Ref" : "HumioLambdaLogLevel" } + } + }, + "VpcConfig" : { + "Fn::If" : [ "ConfigureVPCForIngesterLambdas", + { + "SecurityGroupIds" : { "Ref" : "SecurityGroupIds" }, + "SubnetIds" : { "Ref" : "SubnetIds" } + }, + { + "Ref" : "AWS::NoValue" + } + ] + }, + "Description" : "CloudWatch Metrics Statistics to Humio ingester.", + "Handler" : "metric_statistics_ingester.lambda_handler", + "MemorySize" : "128", + "Role" : { + "Fn::GetAtt" : [ "HumioCloudWatchRole", "Arn" ] + }, + "Runtime" : "python3.8", + "Timeout" : "300" + } + }, + "HumioCloudWatchMetricStatisticsIngesterPermission" : { + "Type" : "AWS::Lambda::Permission", + "Properties" : { + "Action" : "lambda:InvokeFunction", + "FunctionName" : { + "Fn::GetAtt" : [ "HumioCloudWatchMetricStatisticsIngester", "Arn" ] + }, + "Principal" : "logs.amazonaws.com" + } + }, + "HumioCloudWatchMetricStatisticsIngesterLogGroup" : { + "DependsOn" : [ "HumioCloudWatchRole" ], + "Type" : "AWS::Logs::LogGroup", + "Properties" : { + "LogGroupName" : { + "Fn::Join" : [ "", [ "/aws/lambda/", { "Ref" : "HumioCloudWatchMetricStatisticsIngester" } ] ] + }, + "RetentionInDays" : { "Ref" : "HumioLambdaLogRetention" } + } + } + } +} diff --git a/deploy-using-profile.sh b/deploy-using-profile.sh new file mode 100755 index 0000000..f277e17 --- /dev/null +++ b/deploy-using-profile.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e +cat cloudformation.json | jq +make build + +aws s3 cp --acl public-read cloudformation.json s3://humio-public-us-east-1/ --region us-east-1 --profile cloudwatch +aws s3 cp --acl public-read cloudformation-no-trail.json s3://humio-public-us-east-1/ --region us-east-1 --profile cloudwatch + +aws s3 cp --acl public-read target/v1.2.1_cloudwatch2humio.zip s3://humio-public-eu-central-1/ --region eu-central-1 --profile cloudwatch +aws s3 cp --acl public-read target/v1.2.1_cloudwatch2humio.zip s3://humio-public-eu-west-1/ --region eu-west-1 --profile cloudwatch +aws s3 cp --acl public-read target/v1.2.1_cloudwatch2humio.zip s3://humio-public-eu-west-2/ --region eu-west-2 --profile cloudwatch +aws s3 cp --acl public-read target/v1.2.1_cloudwatch2humio.zip s3://humio-public-eu-north-1/ --region eu-north-1 --profile cloudwatch + +aws s3 cp --acl public-read target/v1.2.1_cloudwatch2humio.zip s3://humio-public-us-east-1/ --region us-east-1 --profile cloudwatch +aws s3 cp --acl public-read target/v1.2.1_cloudwatch2humio.zip s3://humio-public-us-east-2/ --region us-east-2 --profile cloudwatch +aws s3 cp --acl public-read target/v1.2.1_cloudwatch2humio.zip s3://humio-public-us-west-2/ --region us-west-2 --profile cloudwatch