Add AWS Elasticsearch as part of infra (#92)

* Add AWS Elasticsearch service as part of staging

* Use latest ES version

* Revert bullet point symbol for README

environments/staging/README.md

@@ -34,6 +34,15 @@ for pre-requisites.
 
 Peer to GDS PDD VPC for VPN access.
 
+## Elasticsearch
+
+### Dependent Modules
+
+- Core
+- VPC Peering
+
+Set-up for creating an Elasticsearch cluster.
+
 ## RDS
 
 Contains an RDS cluster.

environments/staging/elasticsearch/README.md

@@ -0,0 +1,68 @@
+# Staging AWS Elasticsearch service
+
+## Registered `consul` service name
+
+The registered `consul` service name is `elasticsearch`, and the port used is
+`443`.
+
+The actual VPC service and port are registered in `consul`. Any other services
+that require Elasticsearch service should always use the actual VPC service
+name, since the service is hosted under SSL and the SSL certificate to accept is
+registered under the VPC name (and not the `consul` service name).
+
+## Terraform configuration
+
+For `terraform`, you will need something like the following to get the AWS
+Elasticsearch service name:
+
+```hcl
+provider "consul" {
+    version = "~> 1.0"
+    address = "${data.terraform_remote_state.core.consul_api}"
+    scheme = "https"
+    datacenter = "${var.aws_region}"
+}
+
+data "consul_catalog_service" "elasticsearch" {
+    name = "${var.elasticsearch_consul_name}"
+}
+
+locals {
+    elasticsearch_hostname = "${data.consul_catalog_service.elasticsearch.service.0.address}"
+    elasticsearch_port = "${data.consul_catalog_service.elasticsearch.service.0.port}"
+}
+
+variable "aws_region" {
+    default = "ap-southeast-1"
+}
+
+variable "elasticsearch_consul_name" {
+    description = "Elasticsearch registered Consul service name"
+    default = "elasticsearch"
+}
+```
+
+## Other notes
+
+### Terraform operations
+
+You should not require any `vault` or `nomad` token, since the `terraform`
+configuration does not use any of the above modules.
+
+As such, to deploy the service, simple run:
+
+```bash
+terraform apply
+```
+
+If there is no current `terraform` state, then you will need to run:
+
+```bash
+terraform init --backend-config backend-config.tfvars
+terraform workspace new staging
+```
+
+### Kibana
+
+Visualisation tool `kibana` is also packaged together with the AWS Elasticsearch
+service, under the URL `https://<es_vpc_service_name>/_plugin/kibana`.

environments/staging/elasticsearch/backend-config.tfvars

@@ -0,0 +1,4 @@
+bucket = "locus-terraform-state"
+key = "elasticsearch-aws"
+region = "ap-southeast-1"
+dynamodb_table = "locus-terraform-state"

environments/staging/elasticsearch/data.tf

@@ -0,0 +1,11 @@
+# This refers to the "Core" infrastructure deployed. We can refer to resources exposed by
+# the core deployment through this data source
+data "terraform_remote_state" "core" {
+	backend = "s3"
+	workspace = "${terraform.workspace}"
+	config {
+		region = "${var.core_state_region}"
+		bucket = "${var.core_state_bucket}"
+		key = "${var.core_state_key}"
+	}
+}

environments/staging/elasticsearch/es.tf

@@ -0,0 +1,16 @@
+module "es" {
+    source = "github.com/terraform-community-modules/tf_aws_elasticsearch?ref=v0.1.0"
+    domain_name = "${var.es_domain_name}"
+
+    vpc_options = {
+        security_group_ids = ["${aws_security_group.es.id}"]
+        subnet_ids = ["${data.terraform_remote_state.core.vpc_private_subnets[2]}"]
+    }
+
+    instance_count = "${var.es_instance_count}"
+    dedicated_master_type = "${var.es_master_type}"
+    instance_type = "${var.es_instance_type}"
+    es_zone_awareness = false
+    ebs_volume_size = "${var.es_ebs_volume_size}"
+    es_version = "${var.es_version}"
+}

environments/staging/elasticsearch/locals.tf

@@ -0,0 +1,3 @@
+locals {
+    endpoint = "${module.es.endpoint}"
+}

environments/staging/elasticsearch/main.tf

@@ -0,0 +1,15 @@
+terraform {
+    backend "s3" { }
+}
+
+provider "aws" {
+    version = "~> 1.0"
+    region = "${var.aws_region}"
+}
+
+provider "consul" {
+    version = "~> 1.0"
+    address = "${data.terraform_remote_state.core.consul_api}"
+    scheme = "https"
+    datacenter = "${var.aws_region}"
+}

environments/staging/elasticsearch/outputs.tf

@@ -0,0 +1,29 @@
+output "arn" {
+    description = "ARN of the created Elasticsearch domain"
+    value = "${module.es.arn}"
+}
+
+output "domain_id" {
+    description = "Unique identifier for the domain"
+    value = "${module.es.domain_id}"
+}
+
+output "endpoint" {
+    description = "Domain-specific endpoint used to submit index, search, and data upload requests"
+    value = "${local.endpoint}"
+}
+
+output "port" {
+    description = "Elasticsearch service port"
+    value = "${var.es_default_access["port"]}"
+}
+
+output "elasticsearch_url" {
+    description = "Elasticsearch URL"
+    value = "https://${local.endpoint}/"
+}
+
+output "kibana_url" {
+    description = "Kibana URL"
+    value = "https://${local.endpoint}/_plugin/kibana/"
+}

environments/staging/elasticsearch/resources.tf

@@ -0,0 +1,28 @@
+# register Elasticsearch service entry into consul
+resource "consul_catalog_entry" "es" {
+    address = "${local.endpoint}"
+    node = "${var.es_consul_service}"
+
+    service {
+        name = "${var.es_consul_service}"
+        id = "${var.es_consul_service}"
+        port = "${var.es_default_access["port"]}"
+    }
+}
+
+resource "aws_security_group" "es" {
+    name = "${var.security_group_name}"
+    description = "Security group for ${var.security_group_name}"
+    vpc_id = "${data.terraform_remote_state.core.vpc_id}"
+    tags = "${data.terraform_remote_state.core.tags}"
+    revoke_rules_on_delete = true
+}
+
+resource "aws_security_group_rule" "es_access_rule" {
+  type = "${var.es_default_access["type"]}"
+  from_port = "${var.es_default_access["port"]}"
+  to_port = "${var.es_default_access["port"]}"
+  protocol = "${var.es_default_access["protocol"]}"
+  cidr_blocks = "${var.es_access_from_cidr}"
+  security_group_id = "${aws_security_group.es.id}"
+}

environments/staging/elasticsearch/variables.tf

@@ -0,0 +1,85 @@
+#
+# state related
+#
+
+variable "aws_region" {
+    default = "ap-southeast-1"
+}
+
+variable "core_state_region" {
+    description = "AWS Region where the core remote state is stored"
+    default = "ap-southeast-1"
+}
+
+variable "core_state_bucket" {
+    description = "S3 where the core remote state is stored"
+    default = "locus-terraform-state"
+}
+
+variable "core_state_key" {
+    description = "Key where the core remote state is stored"
+    default = "core"
+}
+
+#
+# ES domain related
+#
+
+variable "security_group_name" {
+    description = "Name of security group, leaving this empty generates a group name"
+    default = "l-cloud-es"
+}
+
+variable "es_default_access" {
+    description = "Rest API / Web UI access"
+    type = "map"
+    default = {
+        type = "ingress"
+        protocol = "tcp"
+        port = 443
+    }
+}
+
+variable "es_access_from_cidr" {
+    # via other hosts in l-cloud VPC and direct from GDS AWS VPN
+    description = "Permitted CIDR blocks to access Elasticsearch"
+    default = ["10.158.0.0/16", "172.31.0.0/16"]
+}
+
+variable "es_consul_service" {
+    description = "Name to register in consul to identify Elasticsearch service"
+    default = "elasticsearch"
+}
+
+variable "es_domain_name" {
+    description = "Elasticsearch domain name"
+    default = "l-cloud-es"
+}
+
+variable "es_version" {
+    # Available versions: https://aws.amazon.com/elasticsearch-service/faqs/
+    description = "Elasticsearch version to deploy"
+    default = "6.2"
+}
+
+variable "es_master_type" {
+    # Available types: https://aws.amazon.com/elasticsearch-service/pricing/
+    description = "Elasticsearch instance type for dedicated master node"
+    default = "r4.large.elasticsearch"
+}
+
+variable "es_instance_type" {
+    description = "Elasticsearch instance type for non-master node"
+    default = "r4.large.elasticsearch"
+}
+
+variable "es_instance_count" {
+    # Available types: https://aws.amazon.com/elasticsearch-service/pricing/
+    description = "Number of nodes to be deployed in Elasticsearch"
+    default = "3"
+}
+
+variable "es_ebs_volume_size" {
+    description = "Volume capacity for attached EBS in GB for each node"
+    default = "100"
+}