Add integration tests with one host unreachable

Signed-off-by: Blaine Gardner <[email protected]>

test/Makefile

@@ -17,28 +17,7 @@ build:
 
 setup:
 	@ echo "Setting up test hosts ..."
-	@ # Run some test hosts, and generate a node list with group "one" the first ip, "rest" the
-	@ # rest of the ips and "all" set to all of the ips
-	@ echo '#!/usr/bin/env bash' > $(GROUPFILE)
-	@ echo '' >> $(GROUPFILE)
-	@ for i in $$(seq 1 $(NUM_HOSTS)); do \
-		host=$(HOST_BASENAME)-$$i ; \
-		echo "running test host $$host" ; \
-		docker run --rm --name $$host --hostname $$host --detach $(IMAGE_TAG); \
-		ip=$$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $$host) ; \
-		echo "ip: $$ip" ; \
-		if [ $$i = 1 ] ; then \
-			echo "one='$$ip'" >> $(GROUPFILE) ; \
-			echo "rest='" >> $(GROUPFILE) ; \
-		else \
-			echo $$ip >> $(GROUPFILE) ; \
-		fi ; \
-	done
-	@ echo "'" >> $(GROUPFILE)
-	@ echo 'all="$$one $$rest"' >> $(GROUPFILE)
-	@ echo '' >> $(GROUPFILE)
-	@ echo "  $(GROUPFILE) file:"
-	@ cat $(GROUPFILE)
+	@ ./setup-test-hosts.sh
 
 run:
 	@ ./run-tests.sh

test/run-tests.sh

@@ -28,6 +28,12 @@ run_test 01_config.sh
 run_test 02_run.sh
 run_test 03_copy.sh
 
+# stop the first test host to simulate a node being unreachable
+# host can't be restarted except by creating a new one, so make sure to do this very last
+host=$HOST_BASENAME-1
+docker stop "$host" 1> /dev/null
+run_test 04_node-offline.sh
+
 echo "Test suites run: $num_suites"
 echo "Test failures: $num_failures"
 echo ""

test/setup-test-hosts.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# Run some test hosts, and generate a node list with group "one" the first ip, "rest" the
+# rest of the ips and "all" set to all of the ips
+one=""
+rest=""
+for i in $(seq 1 $NUM_HOSTS); do
+  host=$HOST_BASENAME-$i
+  echo "running test host $host"
+  docker run --rm --name "$host" --hostname "$host" --detach $IMAGE_TAG
+  ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$host")
+  echo "ip: $ip"
+  if [ $i = 1 ] ; then
+    one="$ip"
+  else
+    rest="$(printf '%s\n%s' "$rest" "$ip")"
+  fi
+done
+
+cat << EOF > "$GROUPFILE"
+#!/usr/bin/env bash
+
+one='$one'
+rest='$rest'
+
+EOF
+echo 'all="$one $rest"' >> "$GROUPFILE"
+echo '' >> "$GROUPFILE"
+
+echo "  "$GROUPFILE" file:"
+cat "$GROUPFILE"

test/tests/04_node-offline.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+source tests/shared.sh
+
+echo "Running tests where node 'one' is unreachable ..."
+
+NUM_REMAINING=$(( NUM_HOSTS - 1 ))
+
+assert_retcode "with run command successes on remaining nodes" 1 octopus run -g all 'hostname'
+assert_output_count "$HOST_BASENAME" $(( NUM_REMAINING * 2 ))
+
+assert_success "with no errors running commands on the 'rest' group" octopus run -g rest 'hostname'
+assert_output_count "$HOST_BASENAME" $(( NUM_REMAINING * 2 ))
+
+
+assert_retcode "with copy successes on remaining nodes" 1 \
+  octopus copy -g all -r '$HOME/tests' /tmp/1
+assert_output_count "$HOST_BASENAME" $NUM_REMAINING
+# get md5sums of only files, then get the md5sum of the list of md5sums to keep test output short
+expected_md5sums="$(md5sum test/* 2>/dev/null | awk '{print $1}' | md5sum)"
+assert_success "  and with file md5sums matching" \
+  octopus run -g rest 'md5sum /tmp/1/test/* 2>/dev/null | awk "{print \$1}" | md5sum'
+assert_output_count "$expected_md5sums" $NUM_REMAINING
+
+assert_success "with no errors copying files to the 'rest' group" \
+  octopus copy -g rest -r '$HOME/tests' /tmp/2
+assert_success "  and with file md5sums matching" \
+  octopus run -g rest 'md5sum /tmp/2/test/* 2>/dev/null | awk "{print \$1}" | md5sum'
+assert_output_count "$expected_md5sums" $NUM_REMAINING

test/tests/shared.sh

@@ -34,26 +34,6 @@ function fail () {
   echo ''
 }
 
-function assert_success () { # 1) test name  ...) command
-  echo "  $1"
-  if "${@:2}" &> /tmp/output ; then
-    pass "${@:2}"
-  else
-    cat /tmp/output
-    fail "${@:2}"
-  fi
-}
-
-function assert_failure () { # 1) test name  ...) command
-  echo "  $1"
-  if ! "${@:2}" &> /tmp/output ; then
-    pass "${@:2}"
-  else
-    cat /tmp/output
-    fail "${@:2}"
-  fi
-}
-
 function assert_retcode () { # 1) test name 2) retcode  ...) command
   echo "  $1"
   "${@:3}" &> /tmp/output
@@ -65,6 +45,10 @@ function assert_retcode () { # 1) test name 2) retcode  ...) command
   fi
 }
 
+function assert_success () { # 1) test name  ...) command
+  assert_retcode "$1" 0 "${@:2}"
+}
+
 function assert_output_count () { # 1) output desired 2) desired count
   if [ "$(grep --count "$1" /tmp/output)" == "$2" ]; then
     pass "'$1' is in output exactly $2 times"