Devel (#186)

* Feature: 180 yml file validation

* Feature: 177 leave bastion

* Feature: 176 tagging nodes to ips

commands.go

@@ -29,11 +29,12 @@ const (
 
 var (
 	RegisteredCommands = map[string]cli.CommandFactory{
-		"apply":   ApplyCmdFactory,
-		"destroy": DestroyCmdFactory,
-		"bastion": SshCmdFactory,
-		"node":    NodeSshCmdFactory,
-		"geth":    GethCmdFactory,
+		"apply":    ApplyCmdFactory,
+		"destroy":  DestroyCmdFactory,
+		"bastion":  SshCmdFactory,
+		"node":     NodeSshCmdFactory,
+		"nodeinfo": NodeInfoSshCmdFactory,
+		"geth":     GethCmdFactory,
 	}
 )
 
@@ -54,6 +55,10 @@ type NodeSshCmd struct {
 	SshCmd
 }
 
+type NodeInfoSshCmd struct {
+	SshCmd
+}
+
 type GethCmd struct {
 	SshCmd
 }
@@ -66,6 +71,10 @@ func NodeSshCmdFactory() (command cli.Command, err error) {
 	return NodeSshCmd{}, nil
 }
 
+func NodeInfoSshCmdFactory() (command cli.Command, err error) {
+	return NodeInfoSshCmd{}, nil
+}
+
 func SshCmdFactory() (command cli.Command, err error) {
 	return SshCmd{}, nil
 }
@@ -107,6 +116,12 @@ func (nodeSshCmd NodeSshCmd) Run(args []string) (exitCode int) {
 	return exitCode
 }
 
+func (nodeSshCmd NodeInfoSshCmd) Run(args []string) (exitCode int) {
+	exitCode = nodeSshCmd.getNodeInfo()
+
+	return exitCode
+}
+
 func (sshCmd SshCmd) Run(args []string) (exitCode int) {
 	if runtime.GOOS == "windows" {
 		panic("Cannot run ssh command on Windows")
@@ -295,6 +310,14 @@ func (nodeSshCmd NodeSshCmd) Help() (helpMessage string) {
 	return helpMessage
 }
 
+func (nodeSshCmd NodeInfoSshCmd) Help() (helpMessage string) {
+	helpMessage = "\n This command let you to get detailed info about runing nodes\n"
+	helpMessage = helpMessage + "You must not provide any arguments. There are not arguments in this command\n"
+	helpMessage = helpMessage + "Example: apollo nodeinfo"
+
+	return helpMessage
+}
+
 func (gethCmd GethCmd) Help() (helpMessage string) {
 	helpMessage = "\n This command let you attach via rpc (geth) to certain node\n"
 	helpMessage = helpMessage + "You must provide node number as argument. If number is out of range, ssh will fail\n"
@@ -323,6 +346,11 @@ func (nodeSshCmd NodeSshCmd) Synopsis() (synopsis string) {
 	return synopsis
 }
 
+func (nodeSshCmd NodeInfoSshCmd) Synopsis() (synopsis string) {
+	synopsis = "nodeinfo"
+	return synopsis
+}
+
 func (gethCmd GethCmd) Synopsis() (synopsis string) {
 	synopsis = "geth [number]"
 	return synopsis
@@ -404,3 +432,18 @@ func (sshCmd *SshCmd) runWithScriptLocation(scriptName string, args []string) (e
 
 	return exitCode
 }
+
+func (sshCmd *SshCmd) getNodeInfo() (exitCode int) {
+	bastionSshNodeInfoLocator := "cat /qdata/nodeinfo/ip_* | sort"
+	additionalSshCmdArgs := []string{bastionSshNodeInfoLocator}
+	coreCmd, err := SshCmdFactory()
+
+	if nil != err {
+		fmt.Println(err)
+		return ExitCodeInvalidSetup
+	}
+
+	exitCode = coreCmd.Run(additionalSshCmdArgs)
+
+	return exitCode
+}

commands_test.go

@@ -37,6 +37,13 @@ func TestNodeSshCmdFactory(t *testing.T) {
 	testThatCommandHasWholeInterface(t, command)
 }
 
+func TestNodeInfoSshCmdFactory(t *testing.T) {
+	command, err := NodeInfoSshCmdFactory()
+	assert.Nil(t, err)
+	assert.IsType(t, NodeInfoSshCmd{}, command)
+	testThatCommandHasWholeInterface(t, command)
+}
+
 func TestSshCmdFactory(t *testing.T) {
 	command, err := SshCmdFactory()
 	assert.Nil(t, err)
@@ -129,6 +136,27 @@ func TestNodeSshCmd_RunInvalid(t *testing.T) {
 	terra.TempDirPathLocation = terra.TempDirPath
 }
 
+func TestNodeInfoSshCmd_RunInvalid(t *testing.T) {
+	terra.TempDirPathLocation = ".dummyApolloNodeInfo"
+	dummyFileName := "output.log"
+	deployName := "dummyDeployName"
+	dummyDeployName := terra.TempDirPathLocation + "/" + deployName
+	err := os.MkdirAll(dummyDeployName, 0777)
+	assert.Nil(t, err)
+	PrepareDummyFile(t, dummyDeployName+"/"+dummyFileName, terra.OutputAsAStringWithoutHeaderFixture)
+
+	command, err := NodeInfoSshCmdFactory()
+	assert.Nil(t, err)
+	assert.IsType(t, NodeInfoSshCmd{}, command)
+
+	exitCode := command.Run([]string{})
+	assert.Equal(t, ExitCodeSshDialError, exitCode)
+
+	err = os.RemoveAll(terra.TempDirPathLocation)
+	assert.Nil(t, err)
+	terra.TempDirPathLocation = terra.TempDirPath
+}
+
 func TestSshCmd_RunInvalid(t *testing.T) {
 	terra.TempDirPathLocation = ".dummyApollo"
 	dummyFileName := "output.log"

terra/bastion/main-pantheon.tf

@@ -119,6 +119,12 @@ resource "local_file" "bootstrap" {
 
 set -e
 
+# Kill all running containers
+if [ $(docker ps -a -q | wc -l ) -gt 0 ]; then
+  sudo docker stop $(docker ps -a -q)
+  sudo docker rm $(docker ps -a -q)
+fi
+
 sudo curl -Ls "https://github.com/docker/compose/releases/download/1.24.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
 sudo chmod +x /usr/local/bin/docker-compose
 echo "Pull docker images ..."
@@ -351,7 +357,7 @@ do
 done
 echo ']' >> $target_file
 sudo mv $target_file /opt/ethstats/
-echo '["'${random_id.ethstat_secret.hex}'"]' |  sudo tee -a /opt/ethstats/ws_secret.json
+echo '["'${random_id.ethstat_secret.hex}'"]' |  sudo tee /opt/ethstats/ws_secret.json
 
 # Chainhammer ==================================================
 WORKDIR=/home/admin/chainhammer

terra/bastion/main-parity.tf

@@ -121,6 +121,12 @@ resource "local_file" "bootstrap" {
 
 set -e
 
+# Kill all running containers
+if [ $(docker ps -a -q | wc -l ) -gt 0 ]; then
+  sudo docker stop $(docker ps -a -q)
+  sudo docker rm $(docker ps -a -q)
+fi
+
 sudo curl -Ls "https://github.com/docker/compose/releases/download/1.24.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
 sudo chmod +x /usr/local/bin/docker-compose
 echo "Pull docker images ..."
@@ -350,7 +356,7 @@ do
 done
 echo ']' >> $target_file
 sudo mv $target_file /opt/ethstats/
-echo '["'${random_id.ethstat_secret.hex}'"]' |  sudo tee -a /opt/ethstats/ws_secret.json
+echo '["'${random_id.ethstat_secret.hex}'"]' |  sudo tee /opt/ethstats/ws_secret.json
 
 # Chainhammer ==================================================
 WORKDIR=/home/admin/chainhammer

terra/bastion/main-quorum.tf

@@ -124,6 +124,12 @@ resource "local_file" "bootstrap" {
 
 set -e
 
+# Kill all running containers
+if [ $(docker ps -a -q | wc -l ) -gt 0 ]; then
+  sudo docker stop $(docker ps -a -q)
+  sudo docker rm $(docker ps -a -q)
+fi
+
 sudo curl -Ls "https://github.com/docker/compose/releases/download/1.24.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
 sudo chmod +x /usr/local/bin/docker-compose
 echo "Pull docker images ..."

terra/pantheon/container_definition_bootstrap.tf

@@ -6,6 +6,7 @@ locals {
   hosts_folder         = "${local.shared_volume_container_path}/hosts"
   libfaketime_folder   = "${local.shared_volume_container_path}/lib"
   libfaketime_file     = "${local.shared_volume_container_path}/lib/libfaketime_value"
+  node_info_folder     = "${local.shared_volume_container_path}/nodeinfo"
 
   metadata_bootstrap_container_status_file = "${local.shared_volume_container_path}/metadata_bootstrap_container_status"
 
@@ -95,14 +96,19 @@ locals {
     "echo $HOST_IP > ${local.host_ip_file}",
     "export TASK_ARN=$(curl --connect-timeout 5 --retry 5 --max-time 10 -s $ECS_CONTAINER_METADATA_URI/task | jq -r '.TaskARN')",
     "export REGION=$(echo $TASK_ARN | awk -F: '{ print $4}')",
-    "aws ecs describe-tasks --region $REGION --cluster ${local.ecs_cluster_name} --tasks $TASK_ARN | jq -r '.tasks[0] | .group' > ${local.service_file}",
+    "export SERVICE_GROUP=$(aws ecs describe-tasks --region $REGION --cluster ${local.ecs_cluster_name} --tasks $TASK_ARN | jq -r '.tasks[0] | .group')",
+    "echo $SERVICE_GROUP > ${local.service_file}",
     "mkdir -p ${local.hosts_folder}",
     "mkdir -p ${local.node_ids_folder}",
+    "mkdir -p ${local.node_info_folder}",
     "mkdir -p ${local.accounts_folder}",
     "mkdir -p ${local.libfaketime_folder}",
     "count=0; while [ $count -lt 1 ]; do count=$(ls ${local.libfaketime_folder} | grep libfaketime.so | wc -l); aws s3 cp s3://${local.s3_libfaketime_file} ${local.libfaketime_folder}/libfaketime.so > /dev/null 2>&1 | echo \"Wait for libfaketime to appear on S3 ... \"; sleep 1; done",
     "touch ${local.libfaketime_file}",
-    "aws sqs --region $REGION receive-message --queue-url ${aws_sqs_queue.faketime_queue.id} --visibility-timeout=300 | jq .Messages[].Body | tr -d '\\\"' > ${local.libfaketime_file}",
+    "export CLOCK_SKEW=$(aws sqs --region $REGION receive-message --queue-url ${aws_sqs_queue.faketime_queue.id} --visibility-timeout=300 | jq .Messages[].Body | tr -d '\\\"')",
+    "echo $CLOCK_SKEW > ${local.libfaketime_file}",
+    "echo \"$(echo $SERVICE_GROUP | sed 's/.*://')  ip=$HOST_IP  clock_skew=$CLOCK_SKEW chaos_testing_command=${join(" ", var.chaos_testing_run_command)}\" > ${local.node_info_folder}/${local.normalized_host_ip}",
+    "aws s3 cp ${local.node_info_folder}/${local.normalized_host_ip} s3://${local.s3_revision_folder}/nodeinfo/${local.normalized_host_ip} --sse aws:kms --sse-kms-key-id ${aws_kms_key.bucket.arn}",
     "aws s3 cp ${local.node_id_file} s3://${local.s3_revision_folder}/nodeids/${local.normalized_host_ip} --sse aws:kms --sse-kms-key-id ${aws_kms_key.bucket.arn}",
     "aws s3 cp ${local.host_ip_file} s3://${local.s3_revision_folder}/hosts/${local.normalized_host_ip} --sse aws:kms --sse-kms-key-id ${aws_kms_key.bucket.arn}",
     "aws s3 cp ${local.account_address_file} s3://${local.s3_revision_folder}/accounts/${local.normalized_host_ip} --sse aws:kms --sse-kms-key-id ${aws_kms_key.bucket.arn}",

terra/pantheon/ecs.tf

@@ -1,5 +1,5 @@
 locals {
-  service_name_fmt = "node-%0${min(length(format("%d", var.number_of_nodes)), length(format("%s", var.number_of_nodes))) + 1}d-%s"
+  service_name_fmt = "node-%0${min(length(format("%d", var.number_of_nodes)), length(format("%s", var.number_of_nodes))) + 1}d-%s-%s"
   ecs_cluster_name = "pantheon-network-${var.network_name}"
   pantheon_bucket    = "${var.region}-ecs-${lower(var.network_name)}-${random_id.bucket_postfix.hex}"
 }
@@ -30,7 +30,7 @@ resource "aws_ecs_task_definition" "pantheon" {
 
 resource "aws_ecs_service" "pantheon" {
   count           = "${var.number_of_nodes}"
-  name            = "${format(local.service_name_fmt, count.index + 1, var.network_name)}"
+  name            = "${format(local.service_name_fmt, count.index + 1, var.network_name, aws_ecs_task_definition.pantheon.revision)}"
   cluster         = "${aws_ecs_cluster.pantheon.id}"
   task_definition = "${aws_ecs_task_definition.pantheon.arn}"
   launch_type     = "EC2"

terra/parity/container_definition_bootstrap.tf

@@ -6,6 +6,7 @@ locals {
   hosts_folder         = "${local.shared_volume_container_path}/hosts"
   libfaketime_folder   = "${local.shared_volume_container_path}/lib"
   libfaketime_file     = "${local.shared_volume_container_path}/lib/libfaketime_value"
+  node_info_folder     = "${local.shared_volume_container_path}/nodeinfo"
 
   metadata_bootstrap_container_status_file = "${local.shared_volume_container_path}/metadata_bootstrap_container_status"
 
@@ -95,14 +96,19 @@ locals {
     "echo $HOST_IP > ${local.host_ip_file}",
     "export TASK_ARN=$(curl -s $ECS_CONTAINER_METADATA_URI/task | jq -r '.TaskARN')",
     "export REGION=$(echo $TASK_ARN | awk -F: '{ print $4}')",
-    "aws ecs describe-tasks --region $REGION --cluster ${local.ecs_cluster_name} --tasks $TASK_ARN | jq -r '.tasks[0] | .group' > ${local.service_file}",
+    "export SERVICE_GROUP=$(aws ecs describe-tasks --region $REGION --cluster ${local.ecs_cluster_name} --tasks $TASK_ARN | jq -r '.tasks[0] | .group')",
+    "echo $SERVICE_GROUP > ${local.service_file}",
     "mkdir -p ${local.hosts_folder}",
     "mkdir -p ${local.node_ids_folder}",
+    "mkdir -p ${local.node_info_folder}",
     "mkdir -p ${local.accounts_folder}",
     "mkdir -p ${local.libfaketime_folder}",
     "count=0; while [ $count -lt 1 ]; do count=$(ls ${local.libfaketime_folder} | grep libfaketime.so | wc -l); aws s3 cp s3://${local.s3_libfaketime_file} ${local.libfaketime_folder}/libfaketime.so > /dev/null 2>&1 | echo \"Wait for libfaketime to appear on S3 ... \"; sleep 1; done",
     "touch ${local.libfaketime_file}",
-    "aws sqs --region $REGION receive-message --queue-url ${aws_sqs_queue.faketime_queue.id} --visibility-timeout=300 | jq .Messages[].Body | tr -d '\\\"' > ${local.libfaketime_file}",
+    "export CLOCK_SKEW=$(aws sqs --region $REGION receive-message --queue-url ${aws_sqs_queue.faketime_queue.id} --visibility-timeout=300 | jq .Messages[].Body | tr -d '\\\"')",
+    "echo $CLOCK_SKEW > ${local.libfaketime_file}",
+    "echo \"$(echo $SERVICE_GROUP | sed 's/.*://')  ip=$HOST_IP  clock_skew=$CLOCK_SKEW chaos_testing_command=${join(" ", var.chaos_testing_run_command)}\" > ${local.node_info_folder}/${local.normalized_host_ip}",
+    "aws s3 cp ${local.node_info_folder}/${local.normalized_host_ip} s3://${local.s3_revision_folder}/nodeinfo/${local.normalized_host_ip} --sse aws:kms --sse-kms-key-id ${aws_kms_key.bucket.arn}",
     "aws s3 cp ${local.node_id_file} s3://${local.s3_revision_folder}/nodeids/${local.normalized_host_ip} --sse aws:kms --sse-kms-key-id ${aws_kms_key.bucket.arn}",
     "aws s3 cp ${local.host_ip_file} s3://${local.s3_revision_folder}/hosts/${local.normalized_host_ip} --sse aws:kms --sse-kms-key-id ${aws_kms_key.bucket.arn}",
     "aws s3 cp ${local.account_address_file} s3://${local.s3_revision_folder}/accounts/${local.normalized_host_ip} --sse aws:kms --sse-kms-key-id ${aws_kms_key.bucket.arn}",

terra/parity/ecs.tf

@@ -1,5 +1,5 @@
 locals {
-  service_name_fmt = "node-%0${min(length(format("%d", var.number_of_nodes)), length(format("%s", var.number_of_nodes))) + 1}d-%s"
+  service_name_fmt = "node-%0${min(length(format("%d", var.number_of_nodes)), length(format("%s", var.number_of_nodes))) + 1}d-%s-%s"
   ecs_cluster_name = "parity-network-${var.network_name}"
   parity_bucket    = "${var.region}-ecs-${lower(var.network_name)}-${random_id.bucket_postfix.hex}"
 }
@@ -30,7 +30,7 @@ resource "aws_ecs_task_definition" "parity" {
 
 resource "aws_ecs_service" "parity" {
   count           = "${var.number_of_nodes}"
-  name            = "${format(local.service_name_fmt, count.index + 1, var.network_name)}"
+  name            = "${format(local.service_name_fmt, count.index + 1, var.network_name, aws_ecs_task_definition.parity.revision)}"
   cluster         = "${aws_ecs_cluster.parity.id}"
   task_definition = "${aws_ecs_task_definition.parity.arn}"
   launch_type     = "EC2"

terra/provisioner.go

@@ -1,7 +1,6 @@
 package terra
 
 import (
-	"github.com/hashicorp/terraform/builtin/provisioners/local-exec"
 	"github.com/hashicorp/terraform/builtin/provisioners/remote-exec"
 	"github.com/hashicorp/terraform/terraform"
 )
@@ -11,9 +10,3 @@ func RemoteProvisioner(key string) (returnKey string, provisioner terraform.Reso
 
 	return key, provisioner
 }
-
-func LocalProvisioner(key string) (returnKey string, provisioner terraform.ResourceProvisioner) {
-	provisioner = localexec.Provisioner()
-
-	return key, provisioner
-}

terra/provisioner_test.go

@@ -1,21 +1,14 @@
 package terra
 
 import (
-	"github.com/hashicorp/terraform/builtin/provisioners/local-exec"
+	"github.com/hashicorp/terraform/builtin/provisioners/remote-exec"
 	"github.com/stretchr/testify/assert"
 	"testing"
 )
 
-func TestLocalProvisioner(t *testing.T) {
-	expectedKey := "local-exec"
-	key, provisioner := LocalProvisioner(expectedKey)
-	assert.Equal(t, expectedKey, key)
-	assert.IsType(t, localexec.Provisioner(), provisioner)
-}
-
 func TestRemoteProvisioner(t *testing.T) {
 	expectedKey := "remote-exec"
 	key, provisioner := RemoteProvisioner(expectedKey)
 	assert.Equal(t, expectedKey, key)
-	assert.IsType(t, localexec.Provisioner(), provisioner)
+	assert.IsType(t, remoteexec.Provisioner(), provisioner)
 }

terra/quorum/container_definition_bootstrap.tf

@@ -6,6 +6,7 @@ locals {
   hosts_folder         = "${local.shared_volume_container_path}/hosts"
   libfaketime_folder   = "${local.shared_volume_container_path}/lib"
   libfaketime_file     = "${local.shared_volume_container_path}/lib/libfaketime_value"
+  node_info_folder     = "${local.shared_volume_container_path}/nodeinfo"
 
   metadata_bootstrap_container_status_file = "${local.shared_volume_container_path}/metadata_bootstrap_container_status"
 
@@ -142,14 +143,19 @@ EOP
     "echo $HOST_IP > ${local.host_ip_file}",
     "export TASK_ARN=$(curl -s $ECS_CONTAINER_METADATA_URI/task | jq -r '.TaskARN')",
     "export REGION=$(echo $TASK_ARN | awk -F: '{ print $4}')",
-    "aws ecs describe-tasks --region $REGION --cluster ${local.ecs_cluster_name} --tasks $TASK_ARN | jq -r '.tasks[0] | .group' > ${local.service_file}",
+    "export SERVICE_GROUP=$(aws ecs describe-tasks --region $REGION --cluster ${local.ecs_cluster_name} --tasks $TASK_ARN | jq -r '.tasks[0] | .group')",
+    "echo $SERVICE_GROUP > ${local.service_file}",
     "mkdir -p ${local.hosts_folder}",
     "mkdir -p ${local.node_ids_folder}",
+    "mkdir -p ${local.node_info_folder}",
     "mkdir -p ${local.accounts_folder}",
     "mkdir -p ${local.libfaketime_folder}",
     "count=0; while [ $count -lt 1 ]; do count=$(ls ${local.libfaketime_folder} | grep libfaketime.so | wc -l); aws s3 cp s3://${local.s3_libfaketime_file} ${local.libfaketime_folder}/libfaketime.so > /dev/null 2>&1 | echo \"Wait for libfaketime to appear on S3 ... \"; sleep 1; done",
     "touch ${local.libfaketime_file}",
-    "aws sqs --region $REGION receive-message --queue-url ${aws_sqs_queue.faketime_queue.id} --visibility-timeout=300 | jq .Messages[].Body | tr -d '\\\"' > ${local.libfaketime_file}",
+    "export CLOCK_SKEW=$(aws sqs --region $REGION receive-message --queue-url ${aws_sqs_queue.faketime_queue.id} --visibility-timeout=300 | jq .Messages[].Body | tr -d '\\\"')",
+    "echo $CLOCK_SKEW > ${local.libfaketime_file}",
+    "echo \"$(echo $SERVICE_GROUP | sed 's/.*://')  ip=$HOST_IP  clock_skew=$CLOCK_SKEW chaos_testing_command=${join(" ", var.chaos_testing_run_command)}\" > ${local.node_info_folder}/${local.normalized_host_ip}",
+    "aws s3 cp ${local.node_info_folder}/${local.normalized_host_ip} s3://${local.s3_revision_folder}/nodeinfo/${local.normalized_host_ip} --sse aws:kms --sse-kms-key-id ${aws_kms_key.bucket.arn}",
     "aws s3 cp ${local.node_id_file} s3://${local.s3_revision_folder}/nodeids/${local.normalized_host_ip} --sse aws:kms --sse-kms-key-id ${aws_kms_key.bucket.arn}",
     "aws s3 cp ${local.host_ip_file} s3://${local.s3_revision_folder}/hosts/${local.normalized_host_ip} --sse aws:kms --sse-kms-key-id ${aws_kms_key.bucket.arn}",
     "aws s3 cp ${local.account_address_file} s3://${local.s3_revision_folder}/accounts/${local.normalized_host_ip} --sse aws:kms --sse-kms-key-id ${aws_kms_key.bucket.arn}",

terra/quorum/ecs.tf

@@ -1,5 +1,5 @@
 locals {
-  service_name_fmt = "node-%0${min(length(format("%d", var.number_of_nodes)), length(format("%s", var.number_of_nodes))) + 1}d-%s"
+  service_name_fmt = "node-%0${min(length(format("%d", var.number_of_nodes)), length(format("%s", var.number_of_nodes))) + 1}d-%s-%s"
   ecs_cluster_name = "quorum-network-${var.network_name}"
   quorum_bucket    = "${var.region}-ecs-${lower(var.network_name)}-${random_id.bucket_postfix.hex}"
 }
@@ -31,7 +31,7 @@ resource "aws_ecs_task_definition" "quorum" {
 
 resource "aws_ecs_service" "quorum" {
   count           = "${var.number_of_nodes}"
-  name            = "${format(local.service_name_fmt, count.index + 1, var.network_name)}"
+  name            = "${format(local.service_name_fmt, count.index + 1, var.network_name, aws_ecs_task_definition.quorum.revision)}"
   cluster         = "${aws_ecs_cluster.quorum.id}"
   task_definition = "${aws_ecs_task_definition.quorum.arn}"
   launch_type     = "EC2"