Arc network update (#29058)

* Update arc terraform to allow for coloaction in the default network.Allow usage of reserved ip. Allow usage of existing SA * sync beam env * move aditional runners to load based scaling
apache · Oct 19, 2023 · 12e1933 · 12e1933
1 parent 66c01f0
commit 12e1933
Show file tree

Hide file tree

Showing 10 changed files with 59 additions and 19 deletions.
diff --git a/.github/gh-actions-self-hosted-runners/arc/config/arc_autoscaler.tpl b/.github/gh-actions-self-hosted-runners/arc/config/arc_autoscaler.tpl
@@ -27,7 +27,7 @@ spec:
     name: ${name}
   minReplicas: ${min_runners}
   maxReplicas: ${max_runners}
-  %{~ if webhook_scaling == "true" ~}
+  %{~ if webhook_scaling ~}
   scaleUpTriggers:
     - githubEvent:
         workflowJob: {}

diff --git a/.github/gh-actions-self-hosted-runners/arc/config/arc_deployment.tpl b/.github/gh-actions-self-hosted-runners/arc/config/arc_deployment.tpl
@@ -22,6 +22,9 @@ metadata:
   name: ${name}
 spec:
   template:
+    metadata:
+      annotations:
+        cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
     spec:
       %{~ if selector == true  ~}
       nodeSelector:

diff --git a/.github/gh-actions-self-hosted-runners/arc/environments/beam.env b/.github/gh-actions-self-hosted-runners/arc/environments/beam.env
@@ -20,14 +20,18 @@
 project_id = "apache-beam-testing"
 region = "us-central1"
 zone = "us-central1-b"
-environment = "beam"
+environment = "beam-prod"
 ingress_domain = "action.beam.apache.org" 
 organization = "apache"
 repository = "beam"
 github_app_id_secret_name = "gh-app_id"
 github_app_install_id_secret_name = "gh-app_installation_id"
 github_private_key_secret_name = "gh-pem_key"
 deploy_webhook = "true"
+existing_vpc_name = "default"
+existing_ip_name = "beam-arc-webhook-ip"
+subnetwork_cidr_range = "10.119.0.0/20"
+service_account_id = "[email protected]"
 runner_group = "beam"
 main_runner = {
     name = "main-runner"
@@ -37,7 +41,7 @@ main_runner = {
     max_node_count = "24"
     min_replicas = "1"
     max_replicas = "200"
-    webhook_scaling = true
+    webhook_scaling = false
     disk_size_gb = 200
     requests = {
         cpu = "2"
@@ -52,7 +56,7 @@ additional_runner_pools = [{
     max_node_count = "10"
     min_replicas = "1"
     max_replicas = "10"
-    webhook_scaling = "true"
+    webhook_scaling = false
     requests = {
         cpu = "1500m"
         memory = "5Gi"
@@ -69,7 +73,7 @@ additional_runner_pools = [{
     max_node_count = "10"
     min_replicas = "1"
     max_replicas = "10"
-    webhook_scaling = "true"
+    webhook_scaling = false
     requests = {
         cpu = "7.5"
         memory = "5Gi"

diff --git a/.github/gh-actions-self-hosted-runners/arc/gke.tf b/.github/gh-actions-self-hosted-runners/arc/gke.tf
@@ -21,7 +21,7 @@ resource "google_container_cluster" "actions-runner-gke" {
   project                    = var.project_id
   location                   = var.zone
   initial_node_count         = 1
-  network                    = google_compute_network.actions-runner-network.id
+  network                    = data.google_compute_network.actions-runner-network.id
   subnetwork                 = google_compute_subnetwork.actions-runner-subnetwork.id
   remove_default_node_pool   = true
 
@@ -45,6 +45,7 @@ resource "google_container_node_pool" "main-actions-runner-pool" {
     oauth_scopes = [
       "https://www.googleapis.com/auth/cloud-platform"
     ]
+    service_account = data.google_service_account.service_account.email
     tags = ["actions-runner-pool"]
    }
 }
@@ -72,6 +73,7 @@ resource "google_container_node_pool" "additional_runner_pools" {
     oauth_scopes = [
       "https://www.googleapis.com/auth/cloud-platform"
     ]
+    service_account = data.google_service_account.service_account.email
     tags = ["actions-runner-pool"]
     labels = {
       "runner-pool" = each.value.name
@@ -90,5 +92,15 @@ resource "google_container_node_pool" "additional_runner_pools" {
 
 
 resource "google_compute_global_address" "actions-runner-ip" {
-  name      = "${var.environment}-actions-runner-ip"
+  count = var.deploy_webhook == "true" && var.existing_ip_name == "" ? 1 : 0
+  name  = "${var.environment}-actions-runner-ip"
+}
+
+data "google_compute_global_address" "actions-runner-ip" {
+  count = var.deploy_webhook == "true" ? 1 : 0
+  name  = var.existing_ip_name == "" ? google_compute_global_address.actions-runner-ip[0].name : var.existing_ip_name
+}
+
+data google_service_account "service_account" {
+  account_id = var.service_account_id
 }
diff --git a/.github/gh-actions-self-hosted-runners/arc/kubernetes.tf b/.github/gh-actions-self-hosted-runners/arc/kubernetes.tf
@@ -27,6 +27,7 @@ resource "kubectl_manifest" "arc_autoscaler" {
   depends_on         = [helm_release.arc]
 }
 resource "kubectl_manifest" "arc_webhook_certificate" {
+  count = var.deploy_webhook != "false" ? 1 : 0
   yaml_body          = templatefile("config/arc_certificate.tpl", { ingress_domain = var.ingress_domain })
   override_namespace = "arc"
   depends_on         = [helm_release.arc]

diff --git a/.github/gh-actions-self-hosted-runners/arc/locals.tf b/.github/gh-actions-self-hosted-runners/arc/locals.tf
@@ -19,20 +19,18 @@
 
 
 locals {
-        subnetwork_cidr_range = "10.128.0.0/20"
         arc_values = {
-            "githubWebhookServer.enabled" = "true"
+            "githubWebhookServer.enabled" = "${var.deploy_webhook}"
             "authSecret.create" = "true"
             "authSecret.github_app_id" = data.google_secret_manager_secret_version.github_app_id.secret_data
             "authSecret.github_app_installation_id" = data.google_secret_manager_secret_version.github_app_install_id.secret_data
             "authSecret.github_app_private_key" = data.google_secret_manager_secret_version.github_private_key.secret_data
-            "githubWebhookServer.ingress.enabled" = "true"
+            "githubWebhookServer.ingress.enabled" = "${var.deploy_webhook}"
             "githubWebhookServer.ingress.hosts[0].host" = var.ingress_domain
             "githubWebhookServer.ingress.hosts[0].paths[0].path" = "/"
             "githubWebhookServer.ingress.hosts[0].paths[0].pathType" = "ImplementationSpecific"
             "githubWebhookServer.service.type" = "NodePort"
-            #"githubWebhookServer.ingress.tls[0].hosts[0]" = var.ingress_domain
-            "githubWebhookServer.ingress.annotations.kubernetes\\.io/ingress\\.global-static-ip-name" = google_compute_global_address.actions-runner-ip.name
+            "githubWebhookServer.ingress.annotations.kubernetes\\.io/ingress\\.global-static-ip-name" = var.deploy_webhook != "false" ? data.google_compute_global_address.actions-runner-ip[0].name : "not-configured"
             "githubWebhookServer.ingress.annotations.networking\\.gke\\.io/managed-certificates" = "managed-cert"
             "githubWebhookServer.ingress.annotations.kubernetes\\.io/ingress\\.class" = "gce"
         }

diff --git a/.github/gh-actions-self-hosted-runners/arc/network.tf b/.github/gh-actions-self-hosted-runners/arc/network.tf
@@ -18,15 +18,21 @@
 #
 
 resource "google_compute_network" "actions-runner-network" {
+  count = var.existing_vpc_name == "" ? 1 : 0
   project = var.project_id
   name    = "${var.environment}-actions-runner-network"
   auto_create_subnetworks = false
 }
+data "google_compute_network" "actions-runner-network" {
+  name = var.existing_vpc_name == "" ? google_compute_network.actions-runner-network[0].name : var.existing_vpc_name
+  project = var.project_id
+}
+
 
 resource "google_compute_subnetwork" "actions-runner-subnetwork" {
-  ip_cidr_range            = local.subnetwork_cidr_range
+  ip_cidr_range            = var.subnetwork_cidr_range
   name                     = "${var.environment}-actions-runner-subnetwork"
-  network                  = google_compute_network.actions-runner-network.id
+  network                  = data.google_compute_network.actions-runner-network.id
   region                   = var.region
   project                  = var.project_id
 }
diff --git a/.github/gh-actions-self-hosted-runners/arc/outputs.tf b/.github/gh-actions-self-hosted-runners/arc/outputs.tf
@@ -24,9 +24,8 @@ output "cluster_endpoint" {
     value = google_container_cluster.actions-runner-gke.endpoint
 }
 output "ingress_ip" {
-    value = google_compute_global_address.actions-runner-ip.address
+    value = var.deploy_webhook != "false" ? data.google_compute_global_address.actions-runner-ip[0].address : "Not Configured"
 }
-
 output "get_kubeconfig_command" {
     value = "gcloud container clusters get-credentials ${google_container_cluster.actions-runner-gke.name} --region ${var.zone} --project ${var.project_id}"
 }

diff --git a/.github/gh-actions-self-hosted-runners/arc/provider.tf b/.github/gh-actions-self-hosted-runners/arc/provider.tf
@@ -19,7 +19,7 @@
 
 terraform {
    backend "gcs" {
-    prefix = "test-state"
+    prefix = "prod"
    }
 
   required_providers {
@@ -28,8 +28,8 @@ terraform {
       version = "~> 4.62.0"
     }
     kubectl = {
-      source  = "gavinbunney/kubectl"
-      version = ">= 1.7.0"
+      source  = "alekc/kubectl"
+      version = ">= 2.0.2"
     }
   }
 }

diff --git a/.github/gh-actions-self-hosted-runners/arc/variables.tf b/.github/gh-actions-self-hosted-runners/arc/variables.tf
@@ -58,6 +58,23 @@ variable "deploy_webhook" {
     description = "Enable Github Webhook deployment. use this if the Github App has permissions to create webhooks"
     default = "false"
 }
+variable "existing_vpc_name" {
+    description = "Name of existing VPC to use for deployment"
+    default = ""
+}
+variable "existing_ip_name" {
+    description = "Name of existing IP to use for ingress"
+    default = ""
+}
+variable "subnetwork_cidr_range" { 
+    description = "CIDR range for subnetwork"
+    default = "10.128.0.0/20"
+
+}
+variable "service_account_id" {
+    description = "ID of service account to use for deployment. This can be Name, full Email or Fully Qualified Path"
+    default = ""
+}
 variable "runner_group" {
   description = "value for the runner group label"
   default = ""