From 29e1625f49b8e9adec4534b86ba155623760bbaa Mon Sep 17 00:00:00 2001
From: Steven Miller <sjmiller609@gmail.com>
Date: Mon, 15 Jul 2019 16:53:23 +0000
Subject: [PATCH] productionalizing the terraform code

---
 bastion.tf                    |  8 +++-----
 buckets.tf                    |  1 +
 db.tf                         |  1 +
 examples/from_scratch/main.tf |  1 +
 main.tf                       | 18 ++++++++++++++---
 network.tf                    | 12 ++++++++----
 node_pools.tf                 | 37 ++++++++++++++++++-----------------
 outputs.tf                    | 24 +++++++++++++++++------
 variables.tf                  |  6 ++++++
 9 files changed, 72 insertions(+), 36 deletions(-)

diff --git a/bastion.tf b/bastion.tf
index 5800be9..a3e096b 100644
--- a/bastion.tf
+++ b/bastion.tf
@@ -6,6 +6,9 @@ data "google_compute_image" "ubuntu_lts_latest_image" {
 
 # Bastion host
 resource "google_compute_instance" "bastion" {
+
+  count = var.management_endpoint == "public" ? 0 : 1
+
   name         = local.bastion_name
   machine_type = var.machine_type_bastion
   zone         = local.zone
@@ -25,11 +28,6 @@ resource "google_compute_instance" "bastion" {
     enable-oslogin         = "true"
   }
 
-  service_account {
-    email  = "${google_service_account.bastion.email}"
-    scopes = []
-  }
-
   allow_stopping_for_update = true
 
   metadata_startup_script = <<EOF
diff --git a/buckets.tf b/buckets.tf
index a748dd3..4b9a034 100644
--- a/buckets.tf
+++ b/buckets.tf
@@ -10,5 +10,6 @@ resource "google_storage_bucket" "container_registry" {
   labels = {
     "managed-by" = "terraform"
   }
+
 }
 
diff --git a/db.tf b/db.tf
index 02dada8..50487ab 100644
--- a/db.tf
+++ b/db.tf
@@ -23,6 +23,7 @@ resource "google_sql_database_instance" "instance" {
     create = "30m"
     delete = "30m"
   }
+
 }
 
 resource "random_string" "postgres_airflow_password" {
diff --git a/examples/from_scratch/main.tf b/examples/from_scratch/main.tf
index a4f6a32..23e42b4 100644
--- a/examples/from_scratch/main.tf
+++ b/examples/from_scratch/main.tf
@@ -11,4 +11,5 @@ module "astronomer_gcp" {
   email               = "steven@astronomer.io"
   zonal_cluster       = var.zonal
   management_endpoint = "public"
+  force_destroy       = "DESTROY"
 }
diff --git a/main.tf b/main.tf
index e9d1e10..5b6087b 100644
--- a/main.tf
+++ b/main.tf
@@ -7,6 +7,10 @@ data "http" "local_ip" {
   url = "http://ipv4.icanhazip.com/s"
 }
 
+data "google_container_engine_versions" "versions" {
+  location = var.zonal_cluster ? local.zone : local.region
+}
+
 # GKE cluster
 resource "google_container_cluster" "primary" {
   provider = google-beta
@@ -23,7 +27,14 @@ resource "google_container_cluster" "primary" {
 
   maintenance_policy {
     daily_maintenance_window {
-      start_time = "04:00"
+      # 9am EST
+      # For maintenance windows in general,
+      # people usually choose a time of least-use.
+      # The nature of Airflow is such that the jobs
+      # are likely to run in those same windows, so
+      # it's best to just choose a time where support
+      # will likely be available.
+      start_time = "13:00"
     }
   }
 
@@ -34,8 +45,8 @@ resource "google_container_cluster" "primary" {
   # https://www.terraform.io/docs/providers/google/r/container_cluster.html#node_pool
   location = var.zonal_cluster ? local.zone : local.region
 
-  # min_master_version = local.min_master_version
-  # node_version       = local.node_version
+  min_master_version = data.google_container_engine_versions.versions.latest_master_version
+
   network    = local.core_network_id
   subnetwork = local.gke_subnetwork_id
 
@@ -103,6 +114,7 @@ resource "google_container_cluster" "primary" {
   network_policy {
     enabled = true
   }
+
 }
 
 resource "random_id" "kubeconfig_suffix" {
diff --git a/network.tf b/network.tf
index c852af4..b39ee8c 100644
--- a/network.tf
+++ b/network.tf
@@ -3,6 +3,7 @@
 resource "google_compute_network" "core" {
   name                    = "${var.deployment_id}-core-network"
   auto_create_subnetworks = false
+
 }
 
 #Subnetwork
@@ -23,6 +24,7 @@ resource "google_compute_subnetwork" "gke" {
     range_name    = "${var.deployment_id}-gke-services"
     ip_cidr_range = var.gke_secondary_ip_ranges_services
   }
+
 }
 
 # Router
@@ -34,6 +36,7 @@ resource "google_compute_router" "router" {
   bgp {
     asn = 64514
   }
+
 }
 
 # IP address
@@ -61,25 +64,26 @@ resource "google_compute_router_nat" "nat" {
     name                    = google_compute_subnetwork.bastion.self_link
     source_ip_ranges_to_nat = ["ALL_IP_RANGES"]
   }
+
 }
 
 # https://cloud.google.com/vpc/docs/configure-private-services-access#creating-connection
 resource "google_compute_global_address" "private_ip_address" {
-  provider = google-beta
-
+  provider      = google-beta
   name          = "${var.deployment_id}-private-ip-address"
   purpose       = "VPC_PEERING"
   address_type  = "INTERNAL"
   prefix_length = 16
   network       = google_compute_network.core.self_link
+
 }
 
 resource "google_service_networking_connection" "private_vpc_connection" {
-  provider = google-beta
-
+  provider                = google-beta
   network                 = google_compute_network.core.self_link
   service                 = "servicenetworking.googleapis.com"
   reserved_peering_ranges = [google_compute_global_address.private_ip_address.name]
+
 }
 
 // - Bastion Subnetwork --------------------------------------------------
diff --git a/node_pools.tf b/node_pools.tf
index 5a7fa5d..807263b 100644
--- a/node_pools.tf
+++ b/node_pools.tf
@@ -3,10 +3,11 @@ resource "google_container_node_pool" "node_pool_mt" {
 
   provider = google-beta
 
-  # can't be deleted at the same time.
+  # theses can't be created or deleted at the same time.
   depends_on = [google_container_node_pool.node_pool_platform]
+  version    = data.google_container_engine_versions.versions.latest_master_version
 
-  name = "${var.deployment_id}-node-pool-multi-tenant"
+  name = "${var.deployment_id}-node-pool-multi-tenant-${formatdate("MM-DD-mm", timestamp())}"
 
   # this one can take a long time to delete or create
   timeouts {
@@ -15,19 +16,19 @@ resource "google_container_node_pool" "node_pool_mt" {
   }
 
   lifecycle {
-    # ignore_changes =["node_config[0].labels", "node_config[0].taint"]
-    ignore_changes = ["node_config"]
+    create_before_destroy = true
+    ignore_changes        = [name]
   }
 
   location = var.zonal_cluster ? local.zone : local.region
   cluster  = google_container_cluster.primary.name
 
-  # since we are 'regional' i.e. in 3 zones,
+  # if we are 'regional' i.e. in 3 zones,
   # "1" here means "1 in each zone"
-  initial_node_count = "1"
+  initial_node_count = var.zonal_cluster ? "3" : "1"
 
   autoscaling {
-    min_node_count = "1"
+    min_node_count = var.zonal_cluster ? "3" : "1"
     max_node_count = var.zonal_cluster ? var.max_node_count : ceil(var.max_node_count / 3)
   }
 
@@ -43,12 +44,6 @@ resource "google_container_node_pool" "node_pool_mt" {
 
     machine_type = var.machine_type
 
-    labels = {
-      # One of the pools should have the label indicating that it's
-      # multi-tenant, and the other should not.
-      "astronomer.io/multi-tenant" = "true"
-    }
-
     oauth_scopes = [
       "https://www.googleapis.com/auth/compute",
       "https://www.googleapis.com/auth/devstorage.read_only",
@@ -60,7 +55,7 @@ resource "google_container_node_pool" "node_pool_mt" {
     ]
 
     # this is required for sandbox_config to work
-    image_type = "cos_containerd"
+    image_type = "COS_CONTAINERD"
 
     sandbox_config {
       sandbox_type = "gvisor"
@@ -70,17 +65,18 @@ resource "google_container_node_pool" "node_pool_mt" {
 
 resource "google_container_node_pool" "node_pool_platform" {
 
-  name = "${var.deployment_id}-node-pool-platform"
+  name    = "${var.deployment_id}-node-pool-platform-${formatdate("MM-DD-mm", timestamp())}"
+  version = data.google_container_engine_versions.versions.latest_master_version
 
   location = var.zonal_cluster ? local.zone : local.region
   cluster  = google_container_cluster.primary.name
 
-  # since we are 'regional' i.e. in 3 zones,
+  # if we are 'regional' i.e. in 3 zones,
   # "1" here means "1 in each zone"
-  initial_node_count = "1"
+  initial_node_count = var.zonal_cluster ? "3" : "1"
 
   autoscaling {
-    min_node_count = "1"
+    min_node_count = var.zonal_cluster ? "3" : "1"
     max_node_count = var.zonal_cluster ? var.max_node_count : ceil(var.max_node_count / 3)
   }
 
@@ -109,4 +105,9 @@ resource "google_container_node_pool" "node_pool_platform" {
       "https://www.googleapis.com/auth/trace.append",
     ]
   }
+
+  lifecycle {
+    create_before_destroy = true
+    ignore_changes        = [name]
+  }
 }
diff --git a/outputs.tf b/outputs.tf
index 9aa01d5..fb2ac7d 100644
--- a/outputs.tf
+++ b/outputs.tf
@@ -1,9 +1,5 @@
 output "bastion_proxy_command" {
-  value = "gcloud beta compute ssh --zone ${google_compute_instance.bastion.zone} ${google_compute_instance.bastion.name} --tunnel-through-iap --ssh-flag='-L 1234:127.0.0.1:8888 -C -N'"
-}
-
-output "kubernetes_api_sample_command" {
-  value = "If you have started the api proxy using the bastion SOCKS5 proxy command, this should work:\nhttps_proxy=http://127.0.0.1:1234 kubectl get pods"
+  value = "gcloud beta compute ssh --zone ${google_compute_instance.bastion[0].zone} ${google_compute_instance.bastion[0].name} --tunnel-through-iap --ssh-flag='-L 1234:127.0.0.1:8888 -C -N'"
 }
 
 output "db_connection_string" {
@@ -43,7 +39,19 @@ output "container_registry_bucket_name" {
 }
 
 # https://github.com/hashicorp/terraform/issues/1178
-resource "null_resource" "dependency_setter" {}
+resource "null_resource" "dependency_setter" {
+  depends_on = [google_container_cluster.primary,
+    google_container_node_pool.node_pool_mt,
+  google_container_node_pool.node_pool_platform]
+
+  provisioner "local-exec" {
+    # wait 5 minutes after the first
+    # deployment to allow GKE auto-updates
+    # to converge
+    command = "sleep 300"
+  }
+}
+
 output "depended_on" {
   value = "${null_resource.dependency_setter.id}-${timestamp()}"
 }
@@ -52,3 +60,7 @@ output "gcp_default_service_account_key" {
   value = "${base64decode(google_service_account_key.default_key.private_key)}"
   sensitive = true
 }
+
+output "load_balancer_ip" {
+  value = google_compute_address.nginx_static_ip.address
+}
diff --git a/variables.tf b/variables.tf
index 710a735..9317766 100644
--- a/variables.tf
+++ b/variables.tf
@@ -91,3 +91,9 @@ variable "zonal_cluster" {
 variable "management_endpoint" {
   default = "private"
 }
+
+variable "force_destroy" {
+  default     = ""
+  description = "set to 'DESTROY' to enable destroying operationally critical resources"
+  type        = string
+}