From 74e2fb8886baef9fe5768092cd68579bb889279b Mon Sep 17 00:00:00 2001 From: ytimocin Date: Mon, 29 Apr 2024 13:56:13 -0700 Subject: [PATCH] Adding a job that will purge AWS EKS clusters every 6 hours Signed-off-by: ytimocin --- .github/scripts/delete-old-eks-clusters.py | 68 +++++++++++++++++++ .github/scripts/purge-aws-rds-snapshots.sh | 7 +- .github/workflows/purge-aws-eks-clusters.yaml | 43 ++++++++++++ .../workflows/purge-aws-rds-snapshots.yaml | 4 +- 4 files changed, 116 insertions(+), 6 deletions(-) create mode 100644 .github/scripts/delete-old-eks-clusters.py create mode 100644 .github/workflows/purge-aws-eks-clusters.yaml diff --git a/.github/scripts/delete-old-eks-clusters.py b/.github/scripts/delete-old-eks-clusters.py new file mode 100644 index 00000000..31f5a509 --- /dev/null +++ b/.github/scripts/delete-old-eks-clusters.py @@ -0,0 +1,68 @@ +import boto3 +import os +from datetime import datetime, timezone + + +def delete_node_groups(eks, cluster_name): + node_groups = eks.list_nodegroups(clusterName=cluster_name)['nodegroups'] + for node_group in node_groups: + print(f"Deleting node group {node_group} in cluster {cluster_name}") + eks.delete_nodegroup(clusterName=cluster_name, + nodegroupName=node_group) + + +def delete_fargate_profiles(eks, cluster_name): + fargate_profiles = eks.list_fargate_profiles(clusterName=cluster_name)[ + 'fargateProfileNames'] + for profile in fargate_profiles: + print(f"Deleting Fargate profile {profile} in cluster {cluster_name}") + eks.delete_fargate_profile( + clusterName=cluster_name, fargateProfileName=profile) + + +def delete_old_clusters(prefix, older_than_hours=24): + # Read the region from environment variable + aws_region = os.getenv('AWS_REGION', 'us-west-2') + + # Create an EKS client + eks = boto3.client('eks', region_name=aws_region) + + # List all EKS clusters + clusters = eks.list_clusters()['clusters'] + + # Get the current time + now = datetime.now(timezone.utc) + + # Loop through the clusters + for cluster_name in clusters: + if cluster_name.startswith(prefix): + # Get detailed information about the cluster + cluster_info = eks.describe_cluster(name=cluster_name) + creation_time = cluster_info['cluster']['createdAt'] + + # Calculate the age of the cluster in hours + age = (now - creation_time).total_seconds() / 3600 + + # Delete the cluster if it's older than the specified time + if age > older_than_hours: + print( + f"Deleting EKS cluster {cluster_name}, age: {age:.2f} hours") + + # Delete associated node groups + delete_node_groups(eks, cluster_name) + + # Delete associated Fargate profiles + delete_fargate_profiles(eks, cluster_name) + + # Now delete the cluster + try: + eks.delete_cluster(name=cluster_name) + print(f"Deleted EKS cluster {cluster_name}") + except Exception as e: + print( + f"Failed to delete EKS cluster {cluster_name}: {str(e)}") + + +if __name__ == "__main__": + # Specify the prefix of the clusters you want to target + delete_old_clusters(prefix="eks-samplestest-") diff --git a/.github/scripts/purge-aws-rds-snapshots.sh b/.github/scripts/purge-aws-rds-snapshots.sh index f9efcc03..8068770f 100755 --- a/.github/scripts/purge-aws-rds-snapshots.sh +++ b/.github/scripts/purge-aws-rds-snapshots.sh @@ -4,7 +4,7 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software @@ -16,8 +16,7 @@ set -xe -aws rds describe-db-snapshots --query 'DBSnapshots[].DBSnapshotIdentifier' --output text > snapshots.txt -for rds_snapshot_identifier in $(cat ./snapshots.txt) -do +aws rds describe-db-snapshots --query 'DBSnapshots[].DBSnapshotIdentifier' --output text >snapshots.txt +for rds_snapshot_identifier in $(cat ./snapshots.txt); do aws rds delete-db-snapshot --db-snapshot-identifier $rds_snapshot_identifier done diff --git a/.github/workflows/purge-aws-eks-clusters.yaml b/.github/workflows/purge-aws-eks-clusters.yaml new file mode 100644 index 00000000..ff970e99 --- /dev/null +++ b/.github/workflows/purge-aws-eks-clusters.yaml @@ -0,0 +1,43 @@ +name: Purge AWS EKS Clusters +on: + schedule: + # Runs every 6 hours + - cron: "0 0,6 * * *" + pull_request: + types: [opened, synchronize, reopened] + branches: + - v*.* + - edge +env: + GH_TOKEN: ${{ github.token }} + AWS_REGION: us-west-2 +jobs: + purge_rds_snapshots: + name: Purge AWS EKS Clusters + runs-on: [ubuntu-latest] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.9" + + - name: Install dependencies + run: | + pip install boto3 + + - name: Delete old EKS clusters + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ env.AWS_REGION }} + run: python .github/scripts/delete-old-eks-clusters.py + + - name: Create GitHub issue on failure + if: ${{ failure() && github.event_name != 'pull_request' }} + run: | + gh issue create --title "Purge AWS EKS Clusters workflow failed" \ + --body "Test failed on ${{ github.repository }}. See [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for more details." \ + --repo ${{ github.repository }} diff --git a/.github/workflows/purge-aws-rds-snapshots.yaml b/.github/workflows/purge-aws-rds-snapshots.yaml index 257dd7ba..53a8ddce 100644 --- a/.github/workflows/purge-aws-rds-snapshots.yaml +++ b/.github/workflows/purge-aws-rds-snapshots.yaml @@ -4,8 +4,8 @@ on: # Runs at 00:30 and 12:30 - cron: "30 0,12 * * *" env: - GH_TOKEN: ${{ github.token }} - AWS_REGION: us-west-2 + GH_TOKEN: ${{ github.token }} + AWS_REGION: us-west-2 jobs: purge_rds_snapshots: name: Purge AWS RDS DBInstance snapshots