From e5126810e877894ebbafc4d39d72b0aeca134488 Mon Sep 17 00:00:00 2001 From: Weiguang Date: Mon, 7 May 2018 15:33:21 +0800 Subject: [PATCH] Add AWS Elasticsearch as part of infra (#92) * Add AWS Elasticsearch service as part of staging * Use latest ES version * Revert bullet point symbol for README --- environments/staging/README.md | 9 ++ environments/staging/elasticsearch/README.md | 68 +++++++++++++++ .../elasticsearch/backend-config.tfvars | 4 + environments/staging/elasticsearch/data.tf | 11 +++ environments/staging/elasticsearch/es.tf | 16 ++++ environments/staging/elasticsearch/locals.tf | 3 + environments/staging/elasticsearch/main.tf | 15 ++++ environments/staging/elasticsearch/outputs.tf | 29 +++++++ .../staging/elasticsearch/resources.tf | 28 ++++++ .../staging/elasticsearch/variables.tf | 85 +++++++++++++++++++ 10 files changed, 268 insertions(+) create mode 100644 environments/staging/elasticsearch/README.md create mode 100644 environments/staging/elasticsearch/backend-config.tfvars create mode 100644 environments/staging/elasticsearch/data.tf create mode 100644 environments/staging/elasticsearch/es.tf create mode 100644 environments/staging/elasticsearch/locals.tf create mode 100644 environments/staging/elasticsearch/main.tf create mode 100644 environments/staging/elasticsearch/outputs.tf create mode 100644 environments/staging/elasticsearch/resources.tf create mode 100644 environments/staging/elasticsearch/variables.tf diff --git a/environments/staging/README.md b/environments/staging/README.md index b5864c89..58129ca8 100644 --- a/environments/staging/README.md +++ b/environments/staging/README.md @@ -34,6 +34,15 @@ for pre-requisites. Peer to GDS PDD VPC for VPN access. +## Elasticsearch + +### Dependent Modules + +- Core +- VPC Peering + +Set-up for creating an Elasticsearch cluster. + ## RDS Contains an RDS cluster. diff --git a/environments/staging/elasticsearch/README.md b/environments/staging/elasticsearch/README.md new file mode 100644 index 00000000..4cd98d82 --- /dev/null +++ b/environments/staging/elasticsearch/README.md @@ -0,0 +1,68 @@ +# Staging AWS Elasticsearch service + +## Registered `consul` service name + +The registered `consul` service name is `elasticsearch`, and the port used is +`443`. + +The actual VPC service and port are registered in `consul`. Any other services +that require Elasticsearch service should always use the actual VPC service +name, since the service is hosted under SSL and the SSL certificate to accept is +registered under the VPC name (and not the `consul` service name). + +## Terraform configuration + +For `terraform`, you will need something like the following to get the AWS +Elasticsearch service name: + +```hcl +provider "consul" { + version = "~> 1.0" + address = "${data.terraform_remote_state.core.consul_api}" + scheme = "https" + datacenter = "${var.aws_region}" +} + +data "consul_catalog_service" "elasticsearch" { + name = "${var.elasticsearch_consul_name}" +} + +locals { + elasticsearch_hostname = "${data.consul_catalog_service.elasticsearch.service.0.address}" + elasticsearch_port = "${data.consul_catalog_service.elasticsearch.service.0.port}" +} + +variable "aws_region" { + default = "ap-southeast-1" +} + +variable "elasticsearch_consul_name" { + description = "Elasticsearch registered Consul service name" + default = "elasticsearch" +} +``` + +## Other notes + +### Terraform operations + +You should not require any `vault` or `nomad` token, since the `terraform` +configuration does not use any of the above modules. + +As such, to deploy the service, simple run: + +```bash +terraform apply +``` + +If there is no current `terraform` state, then you will need to run: + +```bash +terraform init --backend-config backend-config.tfvars +terraform workspace new staging +``` + +### Kibana + +Visualisation tool `kibana` is also packaged together with the AWS Elasticsearch +service, under the URL `https:///_plugin/kibana`. diff --git a/environments/staging/elasticsearch/backend-config.tfvars b/environments/staging/elasticsearch/backend-config.tfvars new file mode 100644 index 00000000..8f4fa7db --- /dev/null +++ b/environments/staging/elasticsearch/backend-config.tfvars @@ -0,0 +1,4 @@ +bucket = "locus-terraform-state" +key = "elasticsearch-aws" +region = "ap-southeast-1" +dynamodb_table = "locus-terraform-state" diff --git a/environments/staging/elasticsearch/data.tf b/environments/staging/elasticsearch/data.tf new file mode 100644 index 00000000..d2bb2836 --- /dev/null +++ b/environments/staging/elasticsearch/data.tf @@ -0,0 +1,11 @@ +# This refers to the "Core" infrastructure deployed. We can refer to resources exposed by +# the core deployment through this data source +data "terraform_remote_state" "core" { + backend = "s3" + workspace = "${terraform.workspace}" + config { + region = "${var.core_state_region}" + bucket = "${var.core_state_bucket}" + key = "${var.core_state_key}" + } +} diff --git a/environments/staging/elasticsearch/es.tf b/environments/staging/elasticsearch/es.tf new file mode 100644 index 00000000..b5d860ac --- /dev/null +++ b/environments/staging/elasticsearch/es.tf @@ -0,0 +1,16 @@ +module "es" { + source = "github.com/terraform-community-modules/tf_aws_elasticsearch?ref=v0.1.0" + domain_name = "${var.es_domain_name}" + + vpc_options = { + security_group_ids = ["${aws_security_group.es.id}"] + subnet_ids = ["${data.terraform_remote_state.core.vpc_private_subnets[2]}"] + } + + instance_count = "${var.es_instance_count}" + dedicated_master_type = "${var.es_master_type}" + instance_type = "${var.es_instance_type}" + es_zone_awareness = false + ebs_volume_size = "${var.es_ebs_volume_size}" + es_version = "${var.es_version}" +} diff --git a/environments/staging/elasticsearch/locals.tf b/environments/staging/elasticsearch/locals.tf new file mode 100644 index 00000000..6d2251fe --- /dev/null +++ b/environments/staging/elasticsearch/locals.tf @@ -0,0 +1,3 @@ +locals { + endpoint = "${module.es.endpoint}" +} diff --git a/environments/staging/elasticsearch/main.tf b/environments/staging/elasticsearch/main.tf new file mode 100644 index 00000000..6d9cea14 --- /dev/null +++ b/environments/staging/elasticsearch/main.tf @@ -0,0 +1,15 @@ +terraform { + backend "s3" { } +} + +provider "aws" { + version = "~> 1.0" + region = "${var.aws_region}" +} + +provider "consul" { + version = "~> 1.0" + address = "${data.terraform_remote_state.core.consul_api}" + scheme = "https" + datacenter = "${var.aws_region}" +} diff --git a/environments/staging/elasticsearch/outputs.tf b/environments/staging/elasticsearch/outputs.tf new file mode 100644 index 00000000..31ad99e9 --- /dev/null +++ b/environments/staging/elasticsearch/outputs.tf @@ -0,0 +1,29 @@ +output "arn" { + description = "ARN of the created Elasticsearch domain" + value = "${module.es.arn}" +} + +output "domain_id" { + description = "Unique identifier for the domain" + value = "${module.es.domain_id}" +} + +output "endpoint" { + description = "Domain-specific endpoint used to submit index, search, and data upload requests" + value = "${local.endpoint}" +} + +output "port" { + description = "Elasticsearch service port" + value = "${var.es_default_access["port"]}" +} + +output "elasticsearch_url" { + description = "Elasticsearch URL" + value = "https://${local.endpoint}/" +} + +output "kibana_url" { + description = "Kibana URL" + value = "https://${local.endpoint}/_plugin/kibana/" +} diff --git a/environments/staging/elasticsearch/resources.tf b/environments/staging/elasticsearch/resources.tf new file mode 100644 index 00000000..3b433dd1 --- /dev/null +++ b/environments/staging/elasticsearch/resources.tf @@ -0,0 +1,28 @@ +# register Elasticsearch service entry into consul +resource "consul_catalog_entry" "es" { + address = "${local.endpoint}" + node = "${var.es_consul_service}" + + service { + name = "${var.es_consul_service}" + id = "${var.es_consul_service}" + port = "${var.es_default_access["port"]}" + } +} + +resource "aws_security_group" "es" { + name = "${var.security_group_name}" + description = "Security group for ${var.security_group_name}" + vpc_id = "${data.terraform_remote_state.core.vpc_id}" + tags = "${data.terraform_remote_state.core.tags}" + revoke_rules_on_delete = true +} + +resource "aws_security_group_rule" "es_access_rule" { + type = "${var.es_default_access["type"]}" + from_port = "${var.es_default_access["port"]}" + to_port = "${var.es_default_access["port"]}" + protocol = "${var.es_default_access["protocol"]}" + cidr_blocks = "${var.es_access_from_cidr}" + security_group_id = "${aws_security_group.es.id}" +} diff --git a/environments/staging/elasticsearch/variables.tf b/environments/staging/elasticsearch/variables.tf new file mode 100644 index 00000000..4d3fa928 --- /dev/null +++ b/environments/staging/elasticsearch/variables.tf @@ -0,0 +1,85 @@ +# +# state related +# + +variable "aws_region" { + default = "ap-southeast-1" +} + +variable "core_state_region" { + description = "AWS Region where the core remote state is stored" + default = "ap-southeast-1" +} + +variable "core_state_bucket" { + description = "S3 where the core remote state is stored" + default = "locus-terraform-state" +} + +variable "core_state_key" { + description = "Key where the core remote state is stored" + default = "core" +} + +# +# ES domain related +# + +variable "security_group_name" { + description = "Name of security group, leaving this empty generates a group name" + default = "l-cloud-es" +} + +variable "es_default_access" { + description = "Rest API / Web UI access" + type = "map" + default = { + type = "ingress" + protocol = "tcp" + port = 443 + } +} + +variable "es_access_from_cidr" { + # via other hosts in l-cloud VPC and direct from GDS AWS VPN + description = "Permitted CIDR blocks to access Elasticsearch" + default = ["10.158.0.0/16", "172.31.0.0/16"] +} + +variable "es_consul_service" { + description = "Name to register in consul to identify Elasticsearch service" + default = "elasticsearch" +} + +variable "es_domain_name" { + description = "Elasticsearch domain name" + default = "l-cloud-es" +} + +variable "es_version" { + # Available versions: https://aws.amazon.com/elasticsearch-service/faqs/ + description = "Elasticsearch version to deploy" + default = "6.2" +} + +variable "es_master_type" { + # Available types: https://aws.amazon.com/elasticsearch-service/pricing/ + description = "Elasticsearch instance type for dedicated master node" + default = "r4.large.elasticsearch" +} + +variable "es_instance_type" { + description = "Elasticsearch instance type for non-master node" + default = "r4.large.elasticsearch" +} + +variable "es_instance_count" { + # Available types: https://aws.amazon.com/elasticsearch-service/pricing/ + description = "Number of nodes to be deployed in Elasticsearch" + default = "3" +} + +variable "es_ebs_volume_size" { + description = "Volume capacity for attached EBS in GB for each node" + default = "100" +}