From 12e193370bec7f214328c313d6e97692088c441c Mon Sep 17 00:00:00 2001 From: Vlado Djerek Date: Thu, 19 Oct 2023 15:15:49 +0200 Subject: [PATCH] Arc network update (#29058) * Update arc terraform to allow for coloaction in the default network.Allow usage of reserved ip. Allow usage of existing SA * sync beam env * move aditional runners to load based scaling --- .../arc/config/arc_autoscaler.tpl | 2 +- .../arc/config/arc_deployment.tpl | 3 +++ .../arc/environments/beam.env | 12 ++++++++---- .../gh-actions-self-hosted-runners/arc/gke.tf | 16 ++++++++++++++-- .../arc/kubernetes.tf | 1 + .../arc/locals.tf | 8 +++----- .../arc/network.tf | 10 ++++++++-- .../arc/outputs.tf | 3 +-- .../arc/provider.tf | 6 +++--- .../arc/variables.tf | 17 +++++++++++++++++ 10 files changed, 59 insertions(+), 19 deletions(-) diff --git a/.github/gh-actions-self-hosted-runners/arc/config/arc_autoscaler.tpl b/.github/gh-actions-self-hosted-runners/arc/config/arc_autoscaler.tpl index f6da0aff038a..4b04c5ad8eb1 100644 --- a/.github/gh-actions-self-hosted-runners/arc/config/arc_autoscaler.tpl +++ b/.github/gh-actions-self-hosted-runners/arc/config/arc_autoscaler.tpl @@ -27,7 +27,7 @@ spec: name: ${name} minReplicas: ${min_runners} maxReplicas: ${max_runners} - %{~ if webhook_scaling == "true" ~} + %{~ if webhook_scaling ~} scaleUpTriggers: - githubEvent: workflowJob: {} diff --git a/.github/gh-actions-self-hosted-runners/arc/config/arc_deployment.tpl b/.github/gh-actions-self-hosted-runners/arc/config/arc_deployment.tpl index 6234571c55a3..71f8da3d9df1 100644 --- a/.github/gh-actions-self-hosted-runners/arc/config/arc_deployment.tpl +++ b/.github/gh-actions-self-hosted-runners/arc/config/arc_deployment.tpl @@ -22,6 +22,9 @@ metadata: name: ${name} spec: template: + metadata: + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "false" spec: %{~ if selector == true ~} nodeSelector: diff --git a/.github/gh-actions-self-hosted-runners/arc/environments/beam.env b/.github/gh-actions-self-hosted-runners/arc/environments/beam.env index 9de66b628c89..95c09e1cfad3 100644 --- a/.github/gh-actions-self-hosted-runners/arc/environments/beam.env +++ b/.github/gh-actions-self-hosted-runners/arc/environments/beam.env @@ -20,7 +20,7 @@ project_id = "apache-beam-testing" region = "us-central1" zone = "us-central1-b" -environment = "beam" +environment = "beam-prod" ingress_domain = "action.beam.apache.org" organization = "apache" repository = "beam" @@ -28,6 +28,10 @@ github_app_id_secret_name = "gh-app_id" github_app_install_id_secret_name = "gh-app_installation_id" github_private_key_secret_name = "gh-pem_key" deploy_webhook = "true" +existing_vpc_name = "default" +existing_ip_name = "beam-arc-webhook-ip" +subnetwork_cidr_range = "10.119.0.0/20" +service_account_id = "beam-github-actions@apache-beam-testing.iam.gserviceaccount.com" runner_group = "beam" main_runner = { name = "main-runner" @@ -37,7 +41,7 @@ main_runner = { max_node_count = "24" min_replicas = "1" max_replicas = "200" - webhook_scaling = true + webhook_scaling = false disk_size_gb = 200 requests = { cpu = "2" @@ -52,7 +56,7 @@ additional_runner_pools = [{ max_node_count = "10" min_replicas = "1" max_replicas = "10" - webhook_scaling = "true" + webhook_scaling = false requests = { cpu = "1500m" memory = "5Gi" @@ -69,7 +73,7 @@ additional_runner_pools = [{ max_node_count = "10" min_replicas = "1" max_replicas = "10" - webhook_scaling = "true" + webhook_scaling = false requests = { cpu = "7.5" memory = "5Gi" diff --git a/.github/gh-actions-self-hosted-runners/arc/gke.tf b/.github/gh-actions-self-hosted-runners/arc/gke.tf index bfb048885570..45421ad38b47 100644 --- a/.github/gh-actions-self-hosted-runners/arc/gke.tf +++ b/.github/gh-actions-self-hosted-runners/arc/gke.tf @@ -21,7 +21,7 @@ resource "google_container_cluster" "actions-runner-gke" { project = var.project_id location = var.zone initial_node_count = 1 - network = google_compute_network.actions-runner-network.id + network = data.google_compute_network.actions-runner-network.id subnetwork = google_compute_subnetwork.actions-runner-subnetwork.id remove_default_node_pool = true @@ -45,6 +45,7 @@ resource "google_container_node_pool" "main-actions-runner-pool" { oauth_scopes = [ "https://www.googleapis.com/auth/cloud-platform" ] + service_account = data.google_service_account.service_account.email tags = ["actions-runner-pool"] } } @@ -72,6 +73,7 @@ resource "google_container_node_pool" "additional_runner_pools" { oauth_scopes = [ "https://www.googleapis.com/auth/cloud-platform" ] + service_account = data.google_service_account.service_account.email tags = ["actions-runner-pool"] labels = { "runner-pool" = each.value.name @@ -90,5 +92,15 @@ resource "google_container_node_pool" "additional_runner_pools" { resource "google_compute_global_address" "actions-runner-ip" { - name = "${var.environment}-actions-runner-ip" + count = var.deploy_webhook == "true" && var.existing_ip_name == "" ? 1 : 0 + name = "${var.environment}-actions-runner-ip" +} + +data "google_compute_global_address" "actions-runner-ip" { + count = var.deploy_webhook == "true" ? 1 : 0 + name = var.existing_ip_name == "" ? google_compute_global_address.actions-runner-ip[0].name : var.existing_ip_name +} + +data google_service_account "service_account" { + account_id = var.service_account_id } \ No newline at end of file diff --git a/.github/gh-actions-self-hosted-runners/arc/kubernetes.tf b/.github/gh-actions-self-hosted-runners/arc/kubernetes.tf index bafb653896d7..0a36e1fa2ba6 100644 --- a/.github/gh-actions-self-hosted-runners/arc/kubernetes.tf +++ b/.github/gh-actions-self-hosted-runners/arc/kubernetes.tf @@ -27,6 +27,7 @@ resource "kubectl_manifest" "arc_autoscaler" { depends_on = [helm_release.arc] } resource "kubectl_manifest" "arc_webhook_certificate" { + count = var.deploy_webhook != "false" ? 1 : 0 yaml_body = templatefile("config/arc_certificate.tpl", { ingress_domain = var.ingress_domain }) override_namespace = "arc" depends_on = [helm_release.arc] diff --git a/.github/gh-actions-self-hosted-runners/arc/locals.tf b/.github/gh-actions-self-hosted-runners/arc/locals.tf index 170193b8b6b6..a69d069ab865 100644 --- a/.github/gh-actions-self-hosted-runners/arc/locals.tf +++ b/.github/gh-actions-self-hosted-runners/arc/locals.tf @@ -19,20 +19,18 @@ locals { - subnetwork_cidr_range = "10.128.0.0/20" arc_values = { - "githubWebhookServer.enabled" = "true" + "githubWebhookServer.enabled" = "${var.deploy_webhook}" "authSecret.create" = "true" "authSecret.github_app_id" = data.google_secret_manager_secret_version.github_app_id.secret_data "authSecret.github_app_installation_id" = data.google_secret_manager_secret_version.github_app_install_id.secret_data "authSecret.github_app_private_key" = data.google_secret_manager_secret_version.github_private_key.secret_data - "githubWebhookServer.ingress.enabled" = "true" + "githubWebhookServer.ingress.enabled" = "${var.deploy_webhook}" "githubWebhookServer.ingress.hosts[0].host" = var.ingress_domain "githubWebhookServer.ingress.hosts[0].paths[0].path" = "/" "githubWebhookServer.ingress.hosts[0].paths[0].pathType" = "ImplementationSpecific" "githubWebhookServer.service.type" = "NodePort" - #"githubWebhookServer.ingress.tls[0].hosts[0]" = var.ingress_domain - "githubWebhookServer.ingress.annotations.kubernetes\\.io/ingress\\.global-static-ip-name" = google_compute_global_address.actions-runner-ip.name + "githubWebhookServer.ingress.annotations.kubernetes\\.io/ingress\\.global-static-ip-name" = var.deploy_webhook != "false" ? data.google_compute_global_address.actions-runner-ip[0].name : "not-configured" "githubWebhookServer.ingress.annotations.networking\\.gke\\.io/managed-certificates" = "managed-cert" "githubWebhookServer.ingress.annotations.kubernetes\\.io/ingress\\.class" = "gce" } diff --git a/.github/gh-actions-self-hosted-runners/arc/network.tf b/.github/gh-actions-self-hosted-runners/arc/network.tf index fb7c23a7a3c6..1bc685641337 100644 --- a/.github/gh-actions-self-hosted-runners/arc/network.tf +++ b/.github/gh-actions-self-hosted-runners/arc/network.tf @@ -18,15 +18,21 @@ # resource "google_compute_network" "actions-runner-network" { + count = var.existing_vpc_name == "" ? 1 : 0 project = var.project_id name = "${var.environment}-actions-runner-network" auto_create_subnetworks = false } +data "google_compute_network" "actions-runner-network" { + name = var.existing_vpc_name == "" ? google_compute_network.actions-runner-network[0].name : var.existing_vpc_name + project = var.project_id +} + resource "google_compute_subnetwork" "actions-runner-subnetwork" { - ip_cidr_range = local.subnetwork_cidr_range + ip_cidr_range = var.subnetwork_cidr_range name = "${var.environment}-actions-runner-subnetwork" - network = google_compute_network.actions-runner-network.id + network = data.google_compute_network.actions-runner-network.id region = var.region project = var.project_id } diff --git a/.github/gh-actions-self-hosted-runners/arc/outputs.tf b/.github/gh-actions-self-hosted-runners/arc/outputs.tf index 1e805ca74ce1..f7450911aaf7 100644 --- a/.github/gh-actions-self-hosted-runners/arc/outputs.tf +++ b/.github/gh-actions-self-hosted-runners/arc/outputs.tf @@ -24,9 +24,8 @@ output "cluster_endpoint" { value = google_container_cluster.actions-runner-gke.endpoint } output "ingress_ip" { - value = google_compute_global_address.actions-runner-ip.address + value = var.deploy_webhook != "false" ? data.google_compute_global_address.actions-runner-ip[0].address : "Not Configured" } - output "get_kubeconfig_command" { value = "gcloud container clusters get-credentials ${google_container_cluster.actions-runner-gke.name} --region ${var.zone} --project ${var.project_id}" } diff --git a/.github/gh-actions-self-hosted-runners/arc/provider.tf b/.github/gh-actions-self-hosted-runners/arc/provider.tf index 11aa604fb288..dc557b62a559 100644 --- a/.github/gh-actions-self-hosted-runners/arc/provider.tf +++ b/.github/gh-actions-self-hosted-runners/arc/provider.tf @@ -19,7 +19,7 @@ terraform { backend "gcs" { - prefix = "test-state" + prefix = "prod" } required_providers { @@ -28,8 +28,8 @@ terraform { version = "~> 4.62.0" } kubectl = { - source = "gavinbunney/kubectl" - version = ">= 1.7.0" + source = "alekc/kubectl" + version = ">= 2.0.2" } } } diff --git a/.github/gh-actions-self-hosted-runners/arc/variables.tf b/.github/gh-actions-self-hosted-runners/arc/variables.tf index 43f51938b7d1..3caeffe5a523 100644 --- a/.github/gh-actions-self-hosted-runners/arc/variables.tf +++ b/.github/gh-actions-self-hosted-runners/arc/variables.tf @@ -58,6 +58,23 @@ variable "deploy_webhook" { description = "Enable Github Webhook deployment. use this if the Github App has permissions to create webhooks" default = "false" } +variable "existing_vpc_name" { + description = "Name of existing VPC to use for deployment" + default = "" +} +variable "existing_ip_name" { + description = "Name of existing IP to use for ingress" + default = "" +} +variable "subnetwork_cidr_range" { + description = "CIDR range for subnetwork" + default = "10.128.0.0/20" + +} +variable "service_account_id" { + description = "ID of service account to use for deployment. This can be Name, full Email or Fully Qualified Path" + default = "" +} variable "runner_group" { description = "value for the runner group label" default = ""