From 460ef6564e74d1eab24c7196acf63985e2e4b37b Mon Sep 17 00:00:00 2001 From: Moshe Immermam Date: Wed, 20 Nov 2024 08:37:27 +0200 Subject: [PATCH 1/3] fix: aws ecs health --- pkg/health/health_aws.go | 20 ++--- pkg/health/health_aws_ecs.go | 6 +- pkg/health/health_test.go | 2 +- .../AWS/ECS/Task/running_healthy.yaml | 75 +++++++++++++++++++ 4 files changed, 88 insertions(+), 15 deletions(-) create mode 100644 pkg/health/testdata/AWS/ECS/Task/running_healthy.yaml diff --git a/pkg/health/health_aws.go b/pkg/health/health_aws.go index a66a689..202f5ce 100644 --- a/pkg/health/health_aws.go +++ b/pkg/health/health_aws.go @@ -10,24 +10,24 @@ func GetAWSResourceHealth(_, status string) (health HealthStatus) { } func getAWSHealthByConfigType(configType string, obj map[string]any, states ...string) HealthStatus { - switch configType { - case "AWS::ECS::Task": + switch strings.ToLower(configType) { + case "aws::ecs::task": return GetECSTaskHealth(obj) - case "AWS::Cloudformation::Stack": + case "aws::cloudformation::stack": return GetHealthFromStatusName(get(obj, "StackStatus"), get(obj, "StackStatusReason")) - case "AWS::EC2::Instance": + case "aws::ec2::instance": return GetHealthFromStatusName(get(obj, "State")) - case "AWS::RDS::DBInstance": + case "aws::rds::dbinstance": return GetHealthFromStatusName(get(obj, "DBInstanceStatus")) - case "AWS::ElasticLoadBalancing::LoadBalancer": + case "aws::elasticloadbalancing::loadbalancer": return GetHealthFromStatusName(get(obj, "State", "Code")) - case "AWS::AutoScaling::AutoScalingGroup": + case "aws::autoscaling::autoscalinggroup": return GetHealthFromStatusName(get(obj, "Status")) - case "AWS::Lambda::Function": + case "aws::lambda::function": return GetHealthFromStatusName(get(obj, "State"), get(obj, "StateReasonCode")) - case "AWS::DynamoDB::Table": + case "aws::dynamodb::table": return GetHealthFromStatusName(get(obj, "TableStatus")) - case "AWS::ElastiCache::CacheCluster": + case "aws::elasticache::cachecluster": return GetHealthFromStatusName(get(obj, "CacheClusterStatus")) } diff --git a/pkg/health/health_aws_ecs.go b/pkg/health/health_aws_ecs.go index 1a2892f..dc45dc8 100644 --- a/pkg/health/health_aws_ecs.go +++ b/pkg/health/health_aws_ecs.go @@ -2,18 +2,16 @@ package health import ( "strings" - - "github.com/samber/lo" ) func GetECSTaskHealth(obj map[string]any) (health HealthStatus) { hr := HealthStatus{ - Status: HealthStatusCode(lo.CamelCase(obj["LastStatus"].(string))), + Status: HealthStatusCode(HumanCase(obj["LastStatus"].(string))), Health: HealthUnknown, Ready: false, } - if v, ok := obj["HealthStatus"].(string); ok { + if v, ok := obj["HealthStatus"].(string); ok && v != "UNKNOWN" { hr.Status = HealthStatusCode(HumanCase(v)) } diff --git a/pkg/health/health_test.go b/pkg/health/health_test.go index a8493ab..7f85946 100644 --- a/pkg/health/health_test.go +++ b/pkg/health/health_test.go @@ -61,7 +61,7 @@ func testFixture(t *testing.T, yamlPath string) { } if v, ok := obj.GetAnnotations()["expected-health"]; ok { - assert.Equal(t, v, hr.Health) + assert.Equal(t, health.Health(v), hr.Health) } if v, ok := obj.GetAnnotations()["expected-message"]; ok { diff --git a/pkg/health/testdata/AWS/ECS/Task/running_healthy.yaml b/pkg/health/testdata/AWS/ECS/Task/running_healthy.yaml new file mode 100644 index 0000000..d3419a6 --- /dev/null +++ b/pkg/health/testdata/AWS/ECS/Task/running_healthy.yaml @@ -0,0 +1,75 @@ +annotations: + expected-status: Running + expected-health: healthy +Attachments: + - Id: beb2fa58-3e15-4877-b097-b8e0a4c91f04 + Type: ElasticNetworkInterface + Status: ATTACHED + Details: + - Name: subnetId + Value: subnet-099e59970db9eeb65 + - Name: networkInterfaceId + Value: eni-099e55471b484321b + - Name: macAddress + Value: 0a:f5:69:ba:ad:53 + - Name: privateDnsName + Value: ip-172-31-44-167.eu-west-1.compute.internal + - Name: privateIPv4Address + Value: 172.31.44.167 +Attributes: + - Name: ecs.cpu-architecture + Value: x86_64 + TargetType: "" +AvailabilityZone: eu-west-1b +ClusterArn: arn:aws:ecs:eu-west-1:123456789012:cluster/demo-dev-cluster +Connectivity: CONNECTED +ConnectivityAt: 2024-11-01T17:50:30.827Z +Containers: + - Cpu: "0" + Name: demo-another-container + Image: httpd:latest + GpuIds: [] + TaskArn: arn:aws:ecs:eu-west-1:123456789012:task/demo-dev-cluster/d8f8db0d67e34774935b2fbbc82961a3 + RuntimeId: d8f8db0d67e34774935b2fbbc82961a3-1560160490 + LastStatus: RUNNING + ImageDigest: sha256:bbea29057f25d9543e6a96a8e3cc7c7c937206d20eab2323f478fdb2469d536d + ContainerArn: arn:aws:ecs:eu-west-1:123456789012:container/demo-dev-cluster/d8f8db0d67e34774935b2fbbc82961a3/edf9b73b-b387-480e-954f-2f8dcd531a70 + HealthStatus: UNKNOWN + ManagedAgents: [] + NetworkBindings: [] + NetworkInterfaces: + - AttachmentId: beb2fa58-3e15-4877-b097-b8e0a4c91f04 + PrivateIpv4Address: 172.31.44.167 +Cpu: "512" +CreatedAt: 2024-11-01T17:50:27.075Z +DesiredStatus: RUNNING +EnableExecuteCommand: false +EphemeralStorage: + SizeInGiB: 20 +FargateEphemeralStorage: + SizeInGiB: 20 +Group: service:httpd-service +HealthStatus: UNKNOWN +InferenceAccelerators: [] +LastStatus: RUNNING +LaunchType: FARGATE +Memory: "1024" +Overrides: + ContainerOverrides: + - Name: demo-another-container + Command: [] + Environment: [] + EnvironmentFiles: [] + ResourceRequirements: [] + InferenceAcceleratorOverrides: [] +PlatformFamily: Linux +PlatformVersion: 1.4.0 +PullStartedAt: 2024-11-01T17:50:37.967Z +PullStoppedAt: 2024-11-01T17:50:44.584Z +StartedAt: 2024-11-01T17:50:44.935Z +StartedBy: ecs-svc/4462357315744281090 +StopCode: "" +Tags: [] +TaskArn: arn:aws:ecs:eu-west-1:123456789012:task/demo-dev-cluster/d8f8db0d67e34774935b2fbbc82961a3 +TaskDefinitionArn: arn:aws:ecs:eu-west-1:123456789012:task-definition/demo-httpd-task:1 +Version: 4 From 6145d56d4c8964fdc1204766659648942083c9d8 Mon Sep 17 00:00:00 2001 From: Moshe Immermam Date: Wed, 20 Nov 2024 09:00:38 +0200 Subject: [PATCH 2/3] fix: aws ec2 instance health --- pkg/health/health_aws.go | 6 +- .../AWS/AvailabilityZone/healthy.yaml | 11 + .../testdata/AWS/EC2/Instance/healthy.yaml | 197 ++++++++++++++++++ 3 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 pkg/health/testdata/AWS/AvailabilityZone/healthy.yaml create mode 100644 pkg/health/testdata/AWS/EC2/Instance/healthy.yaml diff --git a/pkg/health/health_aws.go b/pkg/health/health_aws.go index 202f5ce..e22a5b8 100644 --- a/pkg/health/health_aws.go +++ b/pkg/health/health_aws.go @@ -3,6 +3,8 @@ package health import ( "fmt" "strings" + + "github.com/samber/lo" ) func GetAWSResourceHealth(_, status string) (health HealthStatus) { @@ -11,12 +13,14 @@ func GetAWSResourceHealth(_, status string) (health HealthStatus) { func getAWSHealthByConfigType(configType string, obj map[string]any, states ...string) HealthStatus { switch strings.ToLower(configType) { + case "aws::availabilityzone": + return GetHealthFromStatusName(get(obj, "State")) case "aws::ecs::task": return GetECSTaskHealth(obj) case "aws::cloudformation::stack": return GetHealthFromStatusName(get(obj, "StackStatus"), get(obj, "StackStatusReason")) case "aws::ec2::instance": - return GetHealthFromStatusName(get(obj, "State")) + return GetHealthFromStatusName(lo.CoalesceOrEmpty(get(obj, "State"), get(obj, "state"))) case "aws::rds::dbinstance": return GetHealthFromStatusName(get(obj, "DBInstanceStatus")) case "aws::elasticloadbalancing::loadbalancer": diff --git a/pkg/health/testdata/AWS/AvailabilityZone/healthy.yaml b/pkg/health/testdata/AWS/AvailabilityZone/healthy.yaml new file mode 100644 index 0000000..0fcaebf --- /dev/null +++ b/pkg/health/testdata/AWS/AvailabilityZone/healthy.yaml @@ -0,0 +1,11 @@ +annotations: + expected-status: Available +GroupName: eu-west-1 +Messages: [] +NetworkBorderGroup: eu-west-1 +RegionName: eu-west-1 +State: available +ZoneId: euw1-az2 +ZoneName: eu-west-1a +ZoneType: availability-zone +OptInStatus: opt-in-not-required diff --git a/pkg/health/testdata/AWS/EC2/Instance/healthy.yaml b/pkg/health/testdata/AWS/EC2/Instance/healthy.yaml new file mode 100644 index 0000000..9fbfd59 --- /dev/null +++ b/pkg/health/testdata/AWS/EC2/Instance/healthy.yaml @@ -0,0 +1,197 @@ +ami_launch_index: 0 +architecture: x86_64 +block_device_mappings: + - Ebs: + Status: attached + VolumeId: vol-0c9f6ee88cadacb46 + AttachTime: 2024-07-25T08:10:24Z + DeleteOnTermination: true + DeviceName: /dev/xvda +ebs_optimized: false +ena_support: true +hypervisor: xen +iam_instance_profile: arn:aws:iam::765618022540:instance-profile/eks-fac7eebb-3bf4-099a-0ff5-d69508f9990e +image_id: ami-0d0b17734c6a6450b +instance_id: i-070cd36b3b5b5fbe4 +instance_type: t3.xlarge +launch_time: 2024-07-25T08:10:24Z +network_interfaces: + - Groups: + - sg-0fef3b0123b019c30 + - sg-0da304b8a8998ca27 + Status: in-use + OwnerID: "" + AttachTime: 2024-07-25T08:10:24Z + MacAddress: 06:26:bf:8b:b9:49 + Description: "" + DeviceIndex: 0 + AttachmentID: eni-attach-06ac6d4ea3af14395 + Ipv4Prefixes: [] + Ipv6Prefixes: [] + Ipv6Addresses: [] + PrivateDNSName: ip-10-0-4-167.eu-west-1.compute.internal + SourceDestCheck: true + AttachmentStatus: attached + NetworkCardIndex: 0 + PrivateIPAddress: 10.0.4.167 + NetworkInterfaceID: eni-0a6c82200a908be57 + PrivateIPAddresses: + - 10.0.4.167 + - 10.0.4.226 + - 10.0.4.6 + - 10.0.4.72 + - 10.0.4.136 + - 10.0.4.204 + - 10.0.4.79 + - 10.0.4.209 + - 10.0.4.242 + - 10.0.4.56 + - 10.0.4.218 + - 10.0.4.59 + - 10.0.4.221 + - 10.0.4.254 + - 10.0.4.223 + DeleteOnTermination: true + - Groups: + - sg-0fef3b0123b019c30 + - sg-0da304b8a8998ca27 + Status: in-use + OwnerID: "" + AttachTime: 2024-07-25T08:18:20Z + MacAddress: 06:06:46:56:2b:0d + Description: aws-K8S-i-070cd36b3b5b5fbe4 + DeviceIndex: 3 + AttachmentID: eni-attach-09fb5333dccb6593f + Ipv4Prefixes: [] + Ipv6Prefixes: [] + Ipv6Addresses: [] + PrivateDNSName: ip-10-0-4-201.eu-west-1.compute.internal + SourceDestCheck: true + AttachmentStatus: attached + NetworkCardIndex: 0 + PrivateIPAddress: 10.0.4.201 + NetworkInterfaceID: eni-0604f138cb597c7be + PrivateIPAddresses: + - 10.0.4.201 + - 10.0.4.202 + - 10.0.4.42 + - 10.0.4.75 + - 10.0.4.140 + - 10.0.4.109 + - 10.0.4.144 + - 10.0.4.210 + - 10.0.4.50 + - 10.0.4.148 + - 10.0.4.213 + - 10.0.4.247 + - 10.0.4.24 + - 10.0.4.28 + - 10.0.4.158 + DeleteOnTermination: true + - Groups: + - sg-0fef3b0123b019c30 + - sg-0da304b8a8998ca27 + Status: in-use + OwnerID: "" + AttachTime: 2024-07-25T08:11:15Z + MacAddress: 06:0d:76:7f:0a:cb + Description: aws-K8S-i-070cd36b3b5b5fbe4 + DeviceIndex: 1 + AttachmentID: eni-attach-0e420c405e7261313 + Ipv4Prefixes: [] + Ipv6Prefixes: [] + Ipv6Addresses: [] + PrivateDNSName: ip-10-0-4-238.eu-west-1.compute.internal + SourceDestCheck: true + AttachmentStatus: attached + NetworkCardIndex: 0 + PrivateIPAddress: 10.0.4.238 + NetworkInterfaceID: eni-0069456aadcb5ee63 + PrivateIPAddresses: + - 10.0.4.238 + - 10.0.4.160 + - 10.0.4.195 + - 10.0.4.10 + - 10.0.4.12 + - 10.0.4.173 + - 10.0.4.80 + - 10.0.4.177 + - 10.0.4.49 + - 10.0.4.115 + - 10.0.4.152 + - 10.0.4.154 + - 10.0.4.251 + - 10.0.4.126 + - 10.0.4.191 + DeleteOnTermination: true + - Groups: + - sg-0fef3b0123b019c30 + - sg-0da304b8a8998ca27 + Status: in-use + OwnerID: "" + AttachTime: 2024-07-25T08:15:17Z + MacAddress: 06:2c:f1:09:85:47 + Description: aws-K8S-i-070cd36b3b5b5fbe4 + DeviceIndex: 2 + AttachmentID: eni-attach-07cdd5b2d674c2613 + Ipv4Prefixes: [] + Ipv6Prefixes: [] + Ipv6Addresses: [] + PrivateDNSName: ip-10-0-4-32.eu-west-1.compute.internal + SourceDestCheck: true + AttachmentStatus: attached + NetworkCardIndex: 0 + PrivateIPAddress: 10.0.4.32 + NetworkInterfaceID: eni-078d7c3f3620ebfc6 + PrivateIPAddresses: + - 10.0.4.32 + - 10.0.4.196 + - 10.0.4.36 + - 10.0.4.165 + - 10.0.4.43 + - 10.0.4.172 + - 10.0.4.77 + - 10.0.4.205 + - 10.0.4.46 + - 10.0.4.82 + - 10.0.4.51 + - 10.0.4.83 + - 10.0.4.116 + - 10.0.4.86 + - 10.0.4.190 + DeleteOnTermination: true +platform_details: Linux/UNIX +private_dns_name: ip-10-0-4-167.eu-west-1.compute.internal +private_ip_address: 10.0.4.167 +public_dns_name: ec2-52-211-167-57.eu-west-1.compute.amazonaws.com +public_ip_address: 52.211.167.57 +root_device_name: /dev/xvda +root_device_type: ebs +security_groups: + sg-0da304b8a8998ca27: ingress + sg-0fef3b0123b019c30: eks-cluster-sg-mission-control-demo-cluster-1258236895 +source_dest_check: true +state: running +subnet_id: subnet-0e33965610ced8995 +tags: + zone: eu-west-1a + region: eu-west-1 + subnet: subnet-0e33965610ced8995 + account: "765618022540" + network: vpc-09c963a9231399689 + aws:ec2:fleet-id: fleet-029f830f-370e-ee14-8438-af8ad0738c17 + eks:cluster-name: mission-control-demo-cluster + eks:nodegroup-name: ng2 + aws:eks:cluster-name: mission-control-demo-cluster + aws:ec2launchtemplate:id: lt-0f158c6040e332dc9 + aws:autoscaling:groupName: eks-ng2-fac7eebb-3bf4-099a-0ff5-d69508f9990e + aws:ec2launchtemplate:version: "4" + alpha.eksctl.io/nodegroup-name: ng2 + alpha.eksctl.io/nodegroup-type: managed + k8s.io/cluster-autoscaler/enabled: "true" + kubernetes.io/cluster/mission-control-demo-cluster: owned + k8s.io/cluster-autoscaler/mission-control-demo-cluster: owned +usage_operation: RunInstances +usage_operation_update_time: 2024-07-25T08:10:24Z +virtualization_type: hvm +vpc_id: vpc-09c963a9231399689 From ee8838cf564c3fb16de1cff895eefcfe2621cbc8 Mon Sep 17 00:00:00 2001 From: Moshe Immermam Date: Wed, 20 Nov 2024 09:03:16 +0200 Subject: [PATCH 3/3] chore: remove elb test --- .../LoadBalancer/unknown.yaml | 54 ------------------- 1 file changed, 54 deletions(-) delete mode 100644 pkg/health/testdata/AWS/ElasticLoadbalancing/LoadBalancer/unknown.yaml diff --git a/pkg/health/testdata/AWS/ElasticLoadbalancing/LoadBalancer/unknown.yaml b/pkg/health/testdata/AWS/ElasticLoadbalancing/LoadBalancer/unknown.yaml deleted file mode 100644 index 83a086e..0000000 --- a/pkg/health/testdata/AWS/ElasticLoadbalancing/LoadBalancer/unknown.yaml +++ /dev/null @@ -1,54 +0,0 @@ -AvailabilityZones: - - eu-west-1b - - eu-west-1c - - eu-west-1a -BackendServerDescriptions: [] -CanonicalHostedZoneName: ac389ee355c474d20a567c65c12f9d8c-15962110.eu-west-1.elb.amazonaws.com -CanonicalHostedZoneNameID: Z32O12XQLNTSW2 -CreatedTime: 2023-05-10T15:41:50.79Z -DNSName: ac389ee355c474d20a567c65c12f9d8c-15962110.eu-west-1.elb.amazonaws.com -HealthCheck: - Target: TCP:32258 - Timeout: 5 - Interval: 10 - HealthyThreshold: 2 - UnhealthyThreshold: 6 -Instances: - - InstanceId: i-0c0187ae2594844e6 - - InstanceId: i-09b6cd25d7fe29a88 - - InstanceId: i-070cd36b3b5b5fbe4 -ListenerDescriptions: - - Listener: - Protocol: TCP - InstancePort: 32634 - InstanceProtocol: TCP - LoadBalancerPort: 10254 - PolicyNames: [] - - Listener: - Protocol: TCP - InstancePort: 31478 - InstanceProtocol: TCP - LoadBalancerPort: 443 - PolicyNames: [] - - Listener: - Protocol: TCP - InstancePort: 32258 - InstanceProtocol: TCP - LoadBalancerPort: 80 - PolicyNames: [] -LoadBalancerName: ac389ee355c474d20a567c65c12f9d8c -Policies: - OtherPolicies: [] - LBCookieStickinessPolicies: [] - AppCookieStickinessPolicies: [] -Scheme: internet-facing -SecurityGroups: - - sg-0d02d5df08e81e5e0 -SourceSecurityGroup: - GroupName: k8s-elb-ac389ee355c474d20a567c65c12f9d8c - OwnerAlias: "765618022540" -Subnets: - - subnet-017dfefe7e6f269f2 - - subnet-09cba264d3549e8c0 - - subnet-0e33965610ced8995 -VPCId: vpc-09c963a9231399689