From c9ebf31d83bdfc9c8aaebdc2592d0145dd8b44fe Mon Sep 17 00:00:00 2001 From: dfradehubs Date: Thu, 19 Sep 2024 09:28:49 +0200 Subject: [PATCH] feat(Slack): Added better messages to Slack channel --- cmd/main.go | 11 +++++++---- internal/google/mig.go | 34 +++++++++++++++++----------------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 025f007..06e6f4c 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -5,6 +5,7 @@ import ( "elasticsearch-vm-autoscaler/internal/google" "elasticsearch-vm-autoscaler/internal/prometheus" "elasticsearch-vm-autoscaler/internal/slack" + "fmt" "log" "os" @@ -79,7 +80,7 @@ func main() { // If the up condition is met, add a node to the MIG if upCondition { log.Printf("Up condition %s met: Trying to create a new node!", prometheusUpCondition) - err = google.AddNodeToMIG(projectID, zone, migName, debugMode) + currentSize, maxSize, err := google.AddNodeToMIG(projectID, zone, migName, debugMode) if err != nil { log.Printf("Error adding node to MIG: %v", err) time.Sleep(time.Duration(retryIntervalSeconds) * time.Second) @@ -87,11 +88,12 @@ func main() { } // Notify via Slack that a node has been added if slackWebhookURL != "" { - slack.NotifySlack("New node created succesfully in MIG", slackWebhookURL) + message := fmt.Sprintf("Added new node to MIG %s. Current size is %d nodes and the maximum nodes to create are %d", migName, currentSize, maxSize) + slack.NotifySlack(message, slackWebhookURL) } } else if downCondition { // If the down condition is met, remove a node from the MIG log.Printf("Down condition %s met. Trying to remove one node!", prometheusDownCondition) - err = google.RemoveNodeFromMIG(projectID, zone, migName, elasticURL, elasticUser, elasticPassword, debugMode) + currentSize, minSize, nodeRemoved, err := google.RemoveNodeFromMIG(projectID, zone, migName, elasticURL, elasticUser, elasticPassword, debugMode) if err != nil { log.Printf("Error draining node from MIG: %v", err) time.Sleep(time.Duration(retryIntervalSeconds) * time.Second) @@ -99,7 +101,8 @@ func main() { } // Notify via Slack that a node has been removed if slackWebhookURL != "" { - slack.NotifySlack("Removed node from MIG", slackWebhookURL) + message := fmt.Sprintf("Removed node %s from MIG %s. Current size is %d nodes and the minimum nodes to exist are %d", nodeRemoved, migName, currentSize, minSize) + slack.NotifySlack(message, slackWebhookURL) } } else { // No scaling conditions met, so no changes to the MIG diff --git a/internal/google/mig.go b/internal/google/mig.go index aa092f3..8bc43cc 100644 --- a/internal/google/mig.go +++ b/internal/google/mig.go @@ -18,32 +18,32 @@ import ( ) // AddNodeToMIG increases the size of the Managed Instance Group (MIG) by 1, if it has not reached the maximum limit. -func AddNodeToMIG(projectID, zone, migName string, debugMode bool) error { +func AddNodeToMIG(projectID, zone, migName string, debugMode bool) (int32, int32, error) { ctx := context.Background() // Create a new Compute client for managing the MIG client, err := createComputeClient(ctx, compute.NewInstanceGroupManagersRESTClient) if err != nil { - return fmt.Errorf("failed to create Instance Group Managers client: %v", err) + return 0, 0, fmt.Errorf("failed to create Instance Group Managers client: %v", err) } defer client.Close() // Get the current target size of the MIG targetSize, err := getMIGTargetSize(ctx, client, projectID, zone, migName) if err != nil { - return fmt.Errorf("failed to get MIG target size: %v", err) + return 0, 0, fmt.Errorf("failed to get MIG target size: %v", err) } log.Printf("Current size of MIG is %d nodes", targetSize) // Get the scaling limits (minimum and maximum) _, maxSize, err := getMIGScalingLimits() if err != nil { - return fmt.Errorf("failed to get MIG scaling limits: %v", err) + return 0, 0, fmt.Errorf("failed to get MIG scaling limits: %v", err) } // Check if the MIG has reached its maximum size if targetSize >= maxSize { - return fmt.Errorf("MIG has reached its maximum size (%d/%d), no further scaling is possible", targetSize, maxSize) + return 0, 0, fmt.Errorf("MIG has reached its maximum size (%d/%d), no further scaling is possible", targetSize, maxSize) } // Create a request to resize the MIG by increasing the target size by 1 @@ -58,47 +58,47 @@ func AddNodeToMIG(projectID, zone, migName string, debugMode bool) error { if !debugMode { _, err = client.Resize(ctx, req) if err != nil { - return err + return 0, 0, err } else { log.Printf("Scaled up MIG successfully %d/%d", targetSize+1, maxSize) } } - return nil + return targetSize + 1, maxSize, nil } // RemoveNodeFromMIG decreases the size of the Managed Instance Group (MIG) by 1, if it has not reached the minimum limit. -func RemoveNodeFromMIG(projectID, zone, migName, elasticURL, elasticUser, elasticPassword string, debugMode bool) error { +func RemoveNodeFromMIG(projectID, zone, migName, elasticURL, elasticUser, elasticPassword string, debugMode bool) (int32, int32, string, error) { ctx := context.Background() // Create a new Compute client for managing the MIG client, err := createComputeClient(ctx, compute.NewInstanceGroupManagersRESTClient) if err != nil { - return fmt.Errorf("failed to create Instance Group Managers client: %v", err) + return 0, 0, "", fmt.Errorf("failed to create Instance Group Managers client: %v", err) } defer client.Close() // Get the current target size of the MIG targetSize, err := getMIGTargetSize(ctx, client, projectID, zone, migName) if err != nil { - return fmt.Errorf("failed to get MIG target size: %v", err) + return 0, 0, "", fmt.Errorf("failed to get MIG target size: %v", err) } log.Printf("Current size of MIG is %d nodes", targetSize) // Get the scaling limits (minimum and maximum) minSize, _, err := getMIGScalingLimits() if err != nil { - return fmt.Errorf("failed to get MIG scaling limits: %v", err) + return 0, 0, "", fmt.Errorf("failed to get MIG scaling limits: %v", err) } // Check if the MIG has reached its minimum size if targetSize <= minSize { - return fmt.Errorf("MIG has reached its minimum size (%d/%d), no further scaling down is possible", targetSize, minSize) + return 0, 0, "", fmt.Errorf("MIG has reached its minimum size (%d/%d), no further scaling down is possible", targetSize, minSize) } // Get a random instance from the MIG to remove instanceToRemove, err := GetInstanceToRemove(ctx, client, projectID, zone, migName) if err != nil { - return fmt.Errorf("error draining Elasticsearch node: %v", err) + return 0, 0, "", fmt.Errorf("error draining Elasticsearch node: %v", err) } // If not in debug mode, drain the node from Elasticsearch before removal @@ -106,7 +106,7 @@ func RemoveNodeFromMIG(projectID, zone, migName, elasticURL, elasticUser, elasti log.Printf("Instance to remove: %s. Draining from elasticsearch cluster", instanceToRemove) err = elasticsearch.DrainElasticsearchNode(elasticURL, instanceToRemove, elasticUser, elasticPassword) if err != nil { - return fmt.Errorf("error draining Elasticsearch node: %v", err) + return 0, 0, "", fmt.Errorf("error draining Elasticsearch node: %v", err) } log.Printf("Instance drained successfully from elasticsearch cluster") } @@ -126,7 +126,7 @@ func RemoveNodeFromMIG(projectID, zone, migName, elasticURL, elasticUser, elasti if !debugMode { _, err = client.DeleteInstances(ctx, deleteReq) if err != nil { - return fmt.Errorf("error deleting instance: %v", err) + return 0, 0, "", fmt.Errorf("error deleting instance: %v", err) } else { log.Printf("Scaled down MIG successfully %d/%d", targetSize-1, minSize) } @@ -139,12 +139,12 @@ func RemoveNodeFromMIG(projectID, zone, migName, elasticURL, elasticUser, elasti if !debugMode { err = elasticsearch.ClearElasticsearchClusterSettings(elasticURL, elasticUser, elasticPassword) if err != nil { - return fmt.Errorf("error clearing Elasticsearch cluster settings: %v", err) + return 0, 0, "", fmt.Errorf("error clearing Elasticsearch cluster settings: %v", err) } log.Printf("Cleared up elasticsearch settings for draining node") } - return nil + return targetSize - 1, minSize, instanceToRemove, nil } // getMIGScalingLimits retrieves the minimum and maximum scaling limits for a Managed Instance Group (MIG).