Merge pull request #41 from datagovsg/nomad-seperate

Separate out Nomad packer into clients and servers

.gitignore

@@ -1,3 +1,4 @@
 .terraform
 terraform.tfstate.d
 terraform.tfstate.*.backup
+*.retry

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "modules/core/packer/roles/ansible-docker-ubuntu"]
+	path = modules/core/packer/roles/ansible-docker-ubuntu
+	url = https://github.com/lawliet89/ansible-docker-ubuntu.git

environments/staging/README.md

@@ -14,31 +14,39 @@ for pre-requisites.
 
 ### Building AMI with Packer
 
-### Consul
+### Consul servers
 
 ```bash
 packer build \
-    --var-file "packer/consul.json" \
+    --var-file "core/packer/consul.json" \
     "$(git rev-parse --show-toplevel)/modules/core/packer/consul/packer.json"
 ```
 
-#### Consul and Nomad
+#### Nomad Servers
 
 ```bash
 packer build \
-    --var-file "packer/nomad.json" \
-    "$(git rev-parse --show-toplevel)/modules/core/packer/nomad/packer.json"
+    --var-file "core/packer/nomad_servers.json" \
+    "$(git rev-parse --show-toplevel)/modules/core/packer/nomad_servers/packer.json"
 ```
 
-#### Consul and Vault
+#### Nomad Clients
+
+```bash
+packer build \
+    --var-file "core/packer/nomad_clients.json" \
+    "$(git rev-parse --show-toplevel)/modules/core/packer/nomad_clients/packer.json"
+```
+
+<!-- #### Vault Servers
 
 TODO: Setup CA etc.
 
 ```bash
 packer build \
     --var-file "packer/vault.json" \
     "$(git rev-parse --show-toplevel)/modules/core/packer/vault/packer.json"
-```
+``` -->
 
 ## VPC Peering
 

environments/staging/core/main.tf

@@ -31,7 +31,8 @@ module "core" {
     consul_cluster_name = "l-cloud-consul"
 
     // Nomad variables
-    nomad_ami_id = "ami-8e97d8f2"
+    nomad_servers_ami_id = "ami-997638e5"
+    nomad_clients_ami_id = "ami-f9763885"
     nomad_clients_allowed_inbound_cidr_blocks = ["172.31.0.0/16"]
     nomad_cluster_name = "l-cloud-nomad"
     internal_lb_incoming_cidr = ["172.31.0.0/16"]

environments/staging/core/packer/nomad_servers.json

@@ -0,0 +1,4 @@
+{
+    "subnet_id": "subnet-523e3a14",
+    "temporary_security_group_source_cidr": "118.189.84.174/32"
+}

modules/core/example.tfvars

@@ -1,5 +1,6 @@
-nomad_ami_id = "xxx"
 consul_ami_id = "xxx"
+nomad_clients_ami_id = "xxx"
+nomad_servers_ami_id = "xxx"
 consul_allowed_inbound_cidr_blocks = ["0.0.0.0/0"]
 nomad_servers_allowed_inbound_cidr_blocks = ["0.0.0.0/0"]
 nomad_clients_allowed_inbound_cidr_blocks = ["0.0.0.0/0"]

modules/core/nomad_clients.tf

@@ -12,7 +12,7 @@ module "nomad_clients" {
   max_size         = "${var.nomad_clients_max}"
   desired_capacity = "${var.nomad_clients_desired}"
 
-  ami_id = "${var.nomad_ami_id}"
+  ami_id = "${var.nomad_clients_ami_id}"
   user_data = "${data.template_file.user_data_nomad_client.rendered}"
 
   vpc_id = "${module.vpc.vpc_id}"

modules/core/nomad_servers.tf

@@ -5,7 +5,7 @@
 module "nomad_servers" {
   source = "github.com/lawliet89/terraform-aws-nomad//modules/nomad-cluster?ref=aws_autoscaling_attachment"
 
-  ami_id = "${var.nomad_ami_id}"
+  ami_id = "${var.nomad_servers_ami_id}"
 
   cluster_name  = "${var.nomad_cluster_name}-server"
   instance_type = "${var.nomad_server_instance_type}"

modules/core/packer/nomad/README.md → modules/core/packer/nomad_clients/README.md

@@ -1,4 +1,4 @@
-# Consul and Nomad AMI
+# Nomad clients AMI
 
 AMI with Nomad and Consul binaries installed. DNSmasq is also configured to use the local
 Consul agent as its DNS server.

modules/core/packer/nomad/packer.json → modules/core/packer/nomad_clients/packer.json

@@ -1,7 +1,7 @@
 {
     "min_packer_version": "1.1.2",
     "variables": {
-        "ami_base_name": "l-cloud-nomad",
+        "ami_base_name": "l-cloud-nomad-clients",
         "aws_region": "ap-southeast-1",
         "subnet_id": "",
         "temporary_security_group_source_cidr": "0.0.0.0/0",
@@ -15,7 +15,7 @@
     },
     "builders": [
         {
-            "name": "ubuntu-1604-nomad-consul-ami",
+            "name": "ubuntu-1604-nomad-client-ami",
             "ami_name": "{{ user `ami_base_name` }}-{{isotime | clean_ami_name}}",
             "ami_description": "An Ubuntu 16.04 AMI that has Nomad and Consul installed.",
             "instance_type": "t2.micro",

modules/core/packer/nomad_clients/site.yml

@@ -0,0 +1,19 @@
+---
+- name: Provision AMI
+  hosts: all
+  vars:
+    nomad_version: "0.7.0"
+    nomad_module_version: "v0.1.1"
+    consul_module_version: "v0.1.0"
+    consul_version: "1.0.1"
+    docker_version: "17.11.0~ce-0~ubuntu"
+  pre_tasks:
+  - name: Upgrade all packages to the latest version
+    apt:
+      upgrade: yes
+      update_cache: yes
+    become: yes
+  roles:
+  - role: "{{ playbook_dir }}/../roles/consul"
+  - role: "{{ playbook_dir }}/../roles/nomad"
+  - role: "{{ playbook_dir }}/../roles/ansible-docker-ubuntu"

modules/core/packer/nomad_servers/README.md

@@ -0,0 +1,29 @@
+# Nomad servers AMI
+
+AMI with Nomad and Consul binaries installed. DNSmasq is also configured to use the local
+Consul agent as its DNS server.
+
+This is based on this [example](https://github.com/hashicorp/terraform-aws-nomad/tree/master/examples/nomad-consul-ami).
+
+## Configuration Options
+
+See [this page](https://www.packer.io/docs/templates/user-variables.html) for more information.
+
+- `ami_base_name`: Base name for the AMI image. The timestamp will be appended
+- `aws_region`: AWS Region
+- `subnet_id`: ID of subnet to run the builder instance in
+- `temporary_security_group_source_cidr`: Temporary CIDR to allow SSH access from
+- `associate_public_ip_address`: Associate to `true` if the machine provisioned is to be connected via the internet
+- `ssh_interface`: One of `public_ip`, `private_ip`, `public_dns` or `private_dns`. If set, either the public IP address, private IP address, public DNS name or private DNS name will used as the host for SSH. The default behaviour if inside a VPC is to use the public IP address if available, otherwise the private IP address will be used. If not in a VPC the public DNS name will be used.
+- `nomad_version`: Version of Nomad to install
+- `consul_module_version`: Version of the [Terraform Consul](https://github.com/hashicorp/terraform-aws-consul) repository to use
+- `nomad_module_version`: Version of the [Nomad Module](https://github.com/hashicorp/terraform-aws-nomad) to use.
+- `consul_version`: Version of Consul to install
+
+## Building Image
+
+```bash
+packer build \
+    -var-file=vars.json \
+    packer.json
+```

modules/core/packer/nomad_servers/packer.json

@@ -0,0 +1,57 @@
+{
+    "min_packer_version": "1.1.2",
+    "variables": {
+        "ami_base_name": "l-cloud-nomad-servers",
+        "aws_region": "ap-southeast-1",
+        "subnet_id": "",
+        "temporary_security_group_source_cidr": "0.0.0.0/0",
+        "associate_public_ip_address": "true",
+        "ssh_interface": "",
+        "nomad_version": "0.7.1",
+        "consul_module_version": "v0.1.1",
+        "nomad_module_version": "v0.1.1",
+        "consul_version": "1.0.3",
+        "docker_version": "18.02.0~ce-0~ubuntu"
+    },
+    "builders": [
+        {
+            "name": "ubuntu-1604-nomad-server-ami",
+            "ami_name": "{{ user `ami_base_name` }}-{{isotime | clean_ami_name}}",
+            "ami_description": "An Ubuntu 16.04 AMI that has Nomad and Consul installed.",
+            "instance_type": "t2.micro",
+            "region": "{{user `aws_region`}}",
+            "type": "amazon-ebs",
+            "subnet_id": "{{user `subnet_id`}}",
+            "associate_public_ip_address": "{{user `associate_public_ip_address`}}",
+            "ssh_interface": "{{user `ssh_interface`}}",
+            "temporary_security_group_source_cidr": "{{user `temporary_security_group_source_cidr`}}",
+            "source_ami_filter": {
+                "filters": {
+                    "virtualization-type": "hvm",
+                    "architecture": "x86_64",
+                    "name": "*ubuntu-xenial-16.04-amd64-server-*",
+                    "block-device-mapping.volume-type": "gp2",
+                    "root-device-type": "ebs"
+                },
+                "owners": [
+                    "099720109477"
+                ],
+                "most_recent": true
+            },
+            "ssh_username": "ubuntu"
+        }
+    ],
+    "provisioners": [
+        {
+            "type": "ansible",
+            "pause_before": "10s",
+            "playbook_file": "{{ template_dir }}/site.yml",
+            "extra_arguments": [
+                "--extra-vars",
+                "nomad_version={{user `nomad_version`}} consul_module_version={{user `consul_module_version`}} nomad_module_version={{user `nomad_module_version`}} consul_version={{user `consul_version`}} docker_version={{user `docker_version`}}",
+                "-e",
+                "ansible_python_interpreter=/usr/bin/python3"
+            ]
+        }
+    ]
+}

modules/core/packer/nomad/site.yml → modules/core/packer/nomad_servers/site.yml

@@ -6,7 +6,6 @@
     nomad_module_version: "v0.1.1"
     consul_module_version: "v0.1.0"
     consul_version: "1.0.1"
-    docker_version: "17.11.0~ce-0~ubuntu"
   pre_tasks:
   - name: Upgrade all packages to the latest version
     apt:

modules/core/packer/roles/nomad/tasks/main.yml

@@ -30,38 +30,3 @@
     src: "{{ role_path }}/files/config/"
     dest: "/opt/nomad/config"
   become: yes
-- name: Install Docker
-  block:
-  - name: Check dependencies
-    apt:
-      name: "{{ item }}"
-      state: latest
-    with_items:
-      - apt-transport-https
-      - ca-certificates
-      - curl
-      - software-properties-common
-    become: yes
-  - name: Add Docker Key
-    apt_key:
-      url: https://download.docker.com/linux/ubuntu/gpg
-      state: present
-    become: yes
-  - name: Get Ubuntu release name
-    shell: lsb_release -cs
-    register: release_name
-  - name: Add Docker Repositories
-    apt_repository:
-      repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ release_name.stdout }} {{ item }}"
-      state: present
-      update_cache: no
-    with_items:
-      - stable
-      - edge
-    become: yes
-  - name: Install Docker
-    apt:
-      name: "docker-ce={{ docker_version }}"
-      state: present
-      update_cache: yes
-    become: yes

modules/core/variables.tf

@@ -4,12 +4,16 @@
 # You must provide a value for each of these parameters.
 # --------------------------------------------------------------------------------------------------
 
-variable "nomad_ami_id" {
-    description = "AMI ID with Nomad installed"
+variable "nomad_clients_ami_id" {
+    description = "AMI ID for Nomad clients"
+}
+
+variable "nomad_servers_ami_id" {
+    description = "AMI ID for Nomad servers"
 }
 
 variable "consul_ami_id" {
-    description = "AMI ID with Consul installed"
+    description = "AMI ID for Consul servers"
 }
 
 variable "consul_allowed_inbound_cidr_blocks" {