Skip to content

Commit

Permalink
Batiai 2704 fluentbit (#29)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dahoward5
dahoward5 authored Jul 23, 2024
2 parents 2293cb5 + 58ee617 commit c4cf0d0
Show file tree
Hide file tree
Showing 15 changed files with 82 additions and 347 deletions.
27 changes: 17 additions & 10 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,6 @@ The module includes configurations for IAM roles, KMS keys, VPC settings, and va
| `node_taints` | `map(string)` | `{}` | The taints to apply to the EKS nodes. | No |
| `enable_eks_pod_identities` | `bool` | `true` | Enable EKS Pod Identities. | No |
| `pod_identity_tags` | `map(string)` | `{}` | The tags to apply to the Pod Identities. | No |
| `fb_chart_version` | `string` | `"0.1.33"` | Fluent-bit helm chart version. | No |
| `fb_log_encryption` | `bool` | `true` | Enable Fluent-bit log encryption. | No |
| `fb_log_systemd` | `bool` | `true` | Enable Fluent-bit cloudwatch logging for systemd. | No |
| `fb_tags` | `map(string)` | `{}` | The tags to apply to the fluent-bit deployment. | No |
| `fb_log_retention` | `number` | `7` | Days to retain Fluent-bit logs. | No |
| `fb_system_log_retention` | `number` | `7` | Days to retain Fluent-bit systemd logs. | No |
| `fb_drop_namespaces` | `list(string)` | `["kube-system", "cert-manager"]` | Fluent-bit doesn't send logs for these namespaces. | No |
| `fb_kube_namespaces` | `list(string)` | `["kube.*", "cert-manager.*"]` | Kubernetes namespaces. | No |
| `fb_log_filters` | `list(string)` | `["kube-probe", "health", "prometheus", "liveness"]` | Fluent-bit doesn't send logs if message consists of these values. | No |
| `fb_additional_log_filters` | `list(string)` | `["ELB-HealthChecker", "Amazon-Route53-Health-Check-Service"]` | Fluent-bit doesn't send logs if message consists of these values. | No |
| `kp_chart_version` | `string` | `"0.37.0"` | Karpenter helm chart version. | No |
| `karpenter_tags` | `map(string)` | `{}` | The tags to apply to the Karpenter deployment. | No |
| `main_bucket_tags` | `map(string)` | `{}` | The tags to apply to the main bucket. | No |
Expand Down Expand Up @@ -139,6 +129,23 @@ env = "dev"
project = "batcave"
```

3. I am seeing the following error, what does it mean?

```bash
[error] [aws_client] connection initialization error
[error] [output:cloudwatch_logs:cloudwatch_logs.1] Failed to create log stream
[error] [output:cloudwatch_logs:cloudwatch_logs.1] Failed to send events
```

You will see this error on initial stand up of the fluentbit pod(s).
This error should eventually resolve itself as the fluentbit pod(s) come up and start sending logs to CloudWatch.
Look for the following cloudwatch log groups to validate that logs are being sent to cloudwatch as expected:

- `/aws/containerinsights/<cluster_name>/application`
- `/aws/containerinsights/<cluster_name>/dataplane`
- `/aws/containerinsights/<cluster_name>/performance`
- `/aws/containerinsights/<cluster_name>/host`

### Explanation:

1. **Terraform Configuration**:
Expand Down
45 changes: 0 additions & 45 deletions addons/fluentbit.tf
View file

This file was deleted.

33 changes: 0 additions & 33 deletions addons/iam.tf
View file

This file was deleted.

19 changes: 0 additions & 19 deletions addons/settings.tf
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,4 @@
locals {
################################## Fluentbit Settings ##################################
fluentbit_log_name = "${var.eks_cluster_name}-fluent-bit"
fluentbit_namespace = "fluentbit"
fluentbit_service_account_name = "fluent-bit"
fluentbit_system_log_name = "${var.eks_cluster_name}-fluent-bit-systemd"

config_settings = {
log_group_name = local.fluentbit_log_name
system_log_group_name = local.fluentbit_system_log_name
region = var.aws_region
log_retention_days = var.fluentbit_log_retention
drop_namespaces = "(${join("|", var.fluentbit_drop_namespaces)})"
log_filters = "(${join("|", var.fluentbit_log_filters)})"
additional_log_filters = "(${join("|", var.fluentbit_additional_log_filters)})"
kube_namespaces = var.fluentbit_kube_namespaces
}

values = templatefile("${path.module}/values/fluentbit/values.yaml.tpl", local.config_settings)

################################## Karpenter Settings ##################################
karpenter_namespace = "karpenter"
karpenter_service_account_name = "karpenter"
Expand Down
75 changes: 0 additions & 75 deletions addons/values/fluentbit/values.yaml.tpl
View file

This file was deleted.

50 changes: 0 additions & 50 deletions addons/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,56 +109,6 @@ variable "enable_bootstrap_user_data" {
type = bool
}

variable "fluentbit_additional_log_filters" {
description = "Additional log filters to use for Fluentbit"
type = list(string)
}

variable "fluentbit_chart_version" {
description = "The version of the Fluentbit chart to use"
type = string
}

variable "fluentbit_drop_namespaces" {
description = "Namespaces to drop from Fluentbit logs"
type = list(string)
}

variable "fluentbit_kube_namespaces" {
description = "Kubernetes namespaces to use for Fluentbit"
type = list(string)
}

variable "fluentbit_log_encryption" {
description = "Whether to encrypt Fluentbit logs"
type = bool
}

variable "fluentbit_log_filters" {
description = "Log filters to use for Fluentbit"
type = list(string)
}

variable "fluentbit_log_retention" {
description = "The number of days to retain Fluentbit logs"
type = number
}

variable "fluentbit_log_systemd" {
description = "Whether to log systemd messages with Fluentbit"
type = bool
}

variable "fluentbit_system_log_retention" {
description = "The number of days to retain Fluentbit systemd logs"
type = number
}

variable "fluentbit_tags" {
description = "The tags to use for Fluentbit"
type = map(string)
}

variable "gold_image_ami_id" {
description = "The AMI ID to use for the gold image"
type = string
Expand Down
2 changes: 1 addition & 1 deletion efs.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ resource "aws_efs_file_system" "main" {
replication_overwrite = var.efs_protection_replication_overwrite
}

tags = merge(var.efs_tags, { "Name" = "efs-${module.eks.cluster_name}" })
tags = merge(var.efs_tags, local.tags_for_all_resources, { "Name" = "efs-${module.eks.cluster_name}" })
}

resource "aws_efs_mount_target" "main" {
Expand Down
Loading

0 comments on commit c4cf0d0

Please sign in to comment.