Skip to content

Commit

Permalink
Up to date for msk-cluster module (#35)
Browse files Browse the repository at this point in the history
  • Loading branch information
@posquit0
posquit0 authored Nov 8, 2024
1 parent 5f25d5b commit 4791e7f
Show file tree
Hide file tree
Showing 6 changed files with 348 additions and 310 deletions.
52 changes: 27 additions & 25 deletions modules/msk-cluster/cluster.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ resource "aws_msk_configuration" "this" {
# MSK Cluster
###################################################

# TODO: public access cidrs
resource "aws_msk_cluster" "this" {
cluster_name = var.name
kafka_version = var.kafka_version
Expand All @@ -55,10 +54,12 @@ resource "aws_msk_cluster" "this" {
az_distribution = "DEFAULT"
client_subnets = var.broker_subnets
security_groups = concat(
module.security_group[*].id,
[module.security_group.id],
var.broker_additional_security_groups
)

# TODO: `vpc_connectivity`
# TODO: public access cidrs
connectivity_info {
public_access {
type = var.broker_public_access_enabled ? "SERVICE_PROVIDED_EIPS" : "DISABLED"
Expand All @@ -67,52 +68,53 @@ resource "aws_msk_cluster" "this" {

storage_info {
ebs_storage_info {
volume_size = var.broker_volume_size
volume_size = var.broker_storage.volume_size

dynamic "provisioned_throughput" {
for_each = var.broker_volume_provisioned_throughput_enabled ? ["go"] : []
for_each = var.broker_storage.provisioned_throughput.enabled ? [var.broker_storage.provisioned_throughput] : []

content {
enabled = true
volume_throughput = var.broker_volume_provisioned_throughput
enabled = provisioned_throughput.value.enabled
volume_throughput = provisioned_throughput.value.throughput
}
}
}
}
}
storage_mode = var.cluster_storage_mode

configuration_info {
arn = aws_msk_configuration.this.arn
revision = aws_msk_configuration.this.latest_revision
}


## Auth
## Authentiation
client_authentication {
unauthenticated = var.auth_unauthenticated_access_enabled
unauthenticated = var.authentication.unauthenticated_access.enabled

sasl {
iam = var.auth_sasl_iam_enabled
scram = var.auth_sasl_scram_enabled
iam = var.authentication.sasl_iam.enabled
scram = var.authentication.sasl_scram.enabled
}

dynamic "tls" {
for_each = var.auth_tls_enabled ? ["go"] : []
for_each = var.authentication.tls.enabled ? [var.authentication.tls] : []

content {
certificate_authority_arns = var.auth_tls_acm_ca_arns
certificate_authority_arns = tls.value.acm_private_certificate_authorities
}
}
}


## Encryption
encryption_info {
encryption_at_rest_kms_key_arn = var.encryption_at_rest_kms_key
encryption_at_rest_kms_key_arn = var.encryption_at_rest.kms_key

encryption_in_transit {
in_cluster = var.encryption_in_transit_in_cluster_enabled
client_broker = var.encryption_in_transit_client_mode
in_cluster = var.encryption_in_transit.in_cluster_enabled
client_broker = var.encryption_in_transit.client_mode
}
}

Expand All @@ -121,33 +123,33 @@ resource "aws_msk_cluster" "this" {
logging_info {
broker_logs {
cloudwatch_logs {
enabled = var.logging_cloudwatch_enabled
log_group = var.logging_cloudwatch_log_group
enabled = var.logging.cloudwatch_logs.enabled
log_group = var.logging.cloudwatch_logs.log_group
}
firehose {
enabled = var.logging_firehose_enabled
delivery_stream = var.logging_firehose_delivery_stream
enabled = var.logging.firehose.enabled
delivery_stream = var.logging.firehose.delivery_stream
}
s3 {
enabled = var.logging_s3_enabled
bucket = var.logging_s3_bucket
prefix = var.logging_s3_prefix
enabled = var.logging.s3.enabled
bucket = var.logging.s3.bucket
prefix = var.logging.s3.key_prefix
}
}
}


## Monitoring
enhanced_monitoring = var.monitoring_cloudwatch_level
enhanced_monitoring = var.cloudwatch_metrics.monitoring_level

open_monitoring {
prometheus {
jmx_exporter {
enabled_in_broker = var.monitoring_prometheus_jmx_exporter_enabled
enabled_in_broker = var.prometheus.jmx_exporter_enabled
}

node_exporter {
enabled_in_broker = var.monitoring_prometheus_node_exporter_enabled
enabled_in_broker = var.prometheus.node_exporter_enabled
}
}
}
Expand Down
70 changes: 39 additions & 31 deletions modules/msk-cluster/outputs.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,6 @@ output "broker" {
`public_access_enabled` - Whether public access to MSK brokers is enabled.
`security_groups` - A list of the security groups associated with the MSK cluster.
`volume` - A EBS volume information for MSK brokers.
EOF
value = {
size = aws_msk_cluster.this.number_of_broker_nodes
Expand All @@ -56,19 +54,27 @@ output "broker" {
subnets = aws_msk_cluster.this.broker_node_group_info[0].client_subnets
public_access_enabled = var.broker_public_access_enabled
security_groups = aws_msk_cluster.this.broker_node_group_info[0].security_groups
default_security_group_id = try(module.security_group[*].id[0], null)
default_security_group_id = module.security_group.id
}
}

volume = {
size = aws_msk_cluster.this.broker_node_group_info[0].storage_info[0].ebs_storage_info[0].volume_size
provisioned_throughput = {
enabled = try(aws_msk_cluster.this.broker_node_group_info[0].storage_info[0].ebs_storage_info[0].provisioned_throughput[0].enabled, false)
throughput = try(aws_msk_cluster.this.broker_node_group_info[0].storage_info[0].ebs_storage_info[0].provisioned_throughput[0].volume_throughput, null)
}
output "broker_storage" {
description = "The configuration for broker storage of the MSK cluster."
value = {
volume_size = aws_msk_cluster.this.broker_node_group_info[0].storage_info[0].ebs_storage_info[0].volume_size
provisioned_throughput = {
enabled = try(aws_msk_cluster.this.broker_node_group_info[0].storage_info[0].ebs_storage_info[0].provisioned_throughput[0].enabled, false)
throughput = try(aws_msk_cluster.this.broker_node_group_info[0].storage_info[0].ebs_storage_info[0].provisioned_throughput[0].volume_throughput, null)
}
}
}

output "auth" {
output "cluster_storage_mode" {
description = "The storage mode of the MSK cluster."
value = aws_msk_cluster.this.storage_mode
}

output "authentication" {
description = "A configuration for authentication of the Kafka cluster."
value = {
unauthenticated_access = {
Expand All @@ -80,31 +86,33 @@ output "auth" {
}
scram = {
enabled = aws_msk_cluster.this.client_authentication[0].sasl[0].scram
kms_key = var.auth_sasl_scram_kms_key
users = var.auth_sasl_scram_users
kms_key = var.authentication.sasl_scram.kms_key
users = var.authentication.sasl_scram.users
}
}
tls = {
enabled = var.auth_tls_enabled
acm_ca_arns = try(aws_msk_cluster.this.client_authentication[0].tls[0].certificate_authority_arns, [])
enabled = var.authentication.tls.enabled
acm_private_certificate_authorities = try(aws_msk_cluster.this.client_authentication[0].tls[0].certificate_authority_arns, [])
}
}
}

output "encryption" {
output "encryption_at_rest" {
description = <<EOF
A configuration for encryption of the Kafka cluster.
`at_rest` - The configuration for encryption at rest.
`in_transit` - The configuration for encryption in transit.
The configuration for encryption at rest of the Kafka cluster.
EOF
value = {
at_rest = {
kms_key = aws_msk_cluster.this.encryption_info[0].encryption_at_rest_kms_key_arn
}
in_transit = {
in_cluster_enabled = aws_msk_cluster.this.encryption_info[0].encryption_in_transit[0].in_cluster
client_mode = aws_msk_cluster.this.encryption_info[0].encryption_in_transit[0].client_broker
}
kms_key = aws_msk_cluster.this.encryption_info[0].encryption_at_rest_kms_key_arn
}
}

output "encryption_in_transit" {
description = <<EOF
The configuration for encryption in transit of the Kafka cluster.
EOF
value = {
in_cluster_enabled = aws_msk_cluster.this.encryption_info[0].encryption_in_transit[0].in_cluster
client_mode = aws_msk_cluster.this.encryption_info[0].encryption_in_transit[0].client_broker
}
}

Expand All @@ -125,22 +133,22 @@ output "logging" {
delivery_stream = aws_msk_cluster.this.logging_info[0].broker_logs[0].firehose[0].delivery_stream
}
s3 = {
enabled = aws_msk_cluster.this.logging_info[0].broker_logs[0].s3[0].enabled
bucket = aws_msk_cluster.this.logging_info[0].broker_logs[0].s3[0].bucket
prefix = aws_msk_cluster.this.logging_info[0].broker_logs[0].s3[0].prefix
enabled = aws_msk_cluster.this.logging_info[0].broker_logs[0].s3[0].enabled
bucket = aws_msk_cluster.this.logging_info[0].broker_logs[0].s3[0].bucket
key_prefix = aws_msk_cluster.this.logging_info[0].broker_logs[0].s3[0].prefix
}
}
}

output "monitoring" {
description = <<EOF
A configuration for monitoring of the Kafka cluster.
`cloudwatch` - The configuration for MSK CloudWatch Metrics.
`cloudwatch_metrics` - The configuration for MSK CloudWatch Metrics.
`prometheus` - The configuration for Prometheus open monitoring.
EOF
value = {
cloudwatch = {
level = aws_msk_cluster.this.enhanced_monitoring
cloudwatch_metrics = {
monitoring_level = aws_msk_cluster.this.enhanced_monitoring
}
prometheus = {
jmx_exporter_enabled = aws_msk_cluster.this.open_monitoring[0].prometheus[0].jmx_exporter[0].enabled_in_broker
Expand Down
8 changes: 4 additions & 4 deletions modules/msk-cluster/scram-secrets.tf
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
resource "random_password" "this" {
for_each = var.auth_sasl_scram_users
for_each = var.authentication.sasl_scram.users

length = 16

Expand All @@ -19,9 +19,9 @@ resource "random_password" "this" {
# TODO: Create an independant module for msk-scram-users
module "secret" {
source = "tedilabs/secret/aws//modules/secrets-manager-secret"
version = "~> 0.2.0"
version = "~> 0.5.0"

for_each = var.auth_sasl_scram_users
for_each = var.authentication.sasl_scram.users

name = "AmazonMSK_SCRAM/${var.name}/${each.key}"
description = "The SASL/SCRAM secret to provide username and password for MSK cluster authenticaiton."
Expand All @@ -32,7 +32,7 @@ module "secret" {
password = random_password.this[each.key].result
}

kms_key = var.auth_sasl_scram_kms_key
kms_key = var.authentication.sasl_scram.kms_key
policy = null
block_public_policy = true

Expand Down
Loading

0 comments on commit 4791e7f

Please sign in to comment.