test/system: Rework the tests

The tests introduced by commit b5cdc57 have proven to be
rather unstable due to mistakes in their design. The tests were quite
chaotically structured, and because of that images were deleted and
pulled too often, causing several false positives [1, 2].

This changes the structure of the tests in a major way. The tests
(resp. commands) are now run in a manner that better simulates the way
Toolbox is actually used. From a clean state, through creating
containers, using them and in the end deleting them. This should
reduce the strain on the bandwidth and possibly even speed up the
tests themselves.

[1] #372
[2] #374

#375

test/system/001-basics.bats → test/system/001-version.bats

@@ -2,14 +2,6 @@
 
 load helpers
 
-function setup() {
-  :
-}
-
-function teardown() {
-  :
-}
-
 @test "Output version number using full flag" {
   skip "Not implemented"
   run_toolbox --version
@@ -20,7 +12,3 @@ function teardown() {
   run_toolbox version
 }
 
-@test "Show usage screen when no command is given" {
-  run_toolbox 1
-  is "${lines[0]}" "toolbox: missing command" "Usage line 1"
-}

test/system/002-help.bats

@@ -0,0 +1,8 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Show usage screen when no command is given" {
+  run_toolbox 1
+  is "${lines[0]}" "toolbox: missing command" "Usage line 1"
+}

test/system/101-list.bats

@@ -0,0 +1,24 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Run list with zero containers and two images" {
+  run_toolbox list
+  is "${#lines[@]}" "3" "Expected number of lines of the output is 3 (Img: 3 + Spc: 0 + Cont: 0)"
+
+  is "${lines[1]}" ".*registry.fedoraproject.org/.*" "First of the two images"
+  is "${lines[2]}" ".*registry.fedoraproject.org/.*" "Second of the two images"
+}
+
+@test "Run list with zero containers (-c flag)" {
+  run_toolbox list -c
+  is "$output" "" "Output of list should be blank"
+}
+
+@test "Run list with zero images (-i flag)" {
+  run_toolbox list -i
+  is "${#lines[@]}" "3" "Expected number of lines of the output is 3"
+
+  is "${lines[1]}" ".*registry.fedoraproject.org/.*" "First of the two images"
+  is "${lines[2]}" ".*registry.fedoraproject.org/.*" "Second of the two images"
+}

test/system/102-create.bats

@@ -0,0 +1,20 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Create the default container" {
+  run_toolbox -y create
+}
+
+@test "Create a container with a valid custom name ('not-running')" {
+  run_toolbox -y create -c "not-running"
+}
+
+@test "Create a container with a custom image and name ('running';f29)" {
+  run_toolbox -y create -c "running" -i fedora-toolbox:29
+}
+
+@test "Try to create a container with invalid custom name" {
+  run_toolbox 1 -y create -c "ß[email protected]€"
+  is "${lines[0]}" "toolbox: invalid argument for '--container'" "Toolbox reports invalid argument for --container"
+}

test/system/103-list.bats

@@ -0,0 +1,15 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Run list with three containers and two images" {
+  run_toolbox list
+  is "${#lines[@]}" "8" "Expected number of lines of the output is 8 (Img: 3 + Cont: 5 (duplication expected))"
+
+  is "${lines[1]}" ".*registry.fedoraproject.org/.*" "The first of the two images"
+  is "${lines[2]}" ".*registry.fedoraproject.org/.*" "The second of the two images"
+
+  is "${lines[4]}" ".*fedora-toolbox-.*" "The default container should be first in the list"
+  is "${lines[5]}" ".*not-running.*" "The container 'not-running' should be second"
+  is "${lines[6]}" ".*running.*" "The container 'running' should be third (last)"
+}

test/system/201-run.bats

@@ -0,0 +1,23 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Echo 'Hello World' inside of the default container" {
+  run_toolbox run echo "Hello World"
+  is "$output" "Hello World" "Should say 'Hello World'"
+}
+
+@test "Echo 'Hello World' inside of the 'running' container" {
+  run_toolbox run -c running echo "Hello World"
+  is "$output" "Hello World" "Should say 'Hello World'"
+}
+
+@test "Stop the 'running' container using 'podman stop'" {
+  run_podman stop running
+  is "${#lines[@]}" "1" "Expected number of lines of the output is 1 (with the id of the container)"
+}
+
+@test "Echo 'hello World' again in the 'running' container after being stopped and exit" {
+  run_toolbox run -c running echo "Hello World"
+  is "$output" "Hello World" "Should say 'Hello World'"
+}

test/system/301-rm.bats

@@ -0,0 +1,28 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Try to remove a nonexistent container" {
+  run_toolbox 1 rm nonexistentcontainer
+  is "$output" "toolbox: failed to inspect $todelete" "Toolbox should fail with: no such container"
+}
+
+@test "Try to remove the running container 'running'" {
+  run_toolbox 1 rm running
+  is "$output" "toolbox: failed to remove container running" "Toolbox should fail to remove the running container"
+}
+
+@test "Remove the not running container 'not-running'" {
+  run_toolbox rm not-running
+  is "$output" "" "The output should be empty"
+}
+
+@test "Force remove the running container 'running'" {
+  run_toolbox rm --force running
+  is "$output" "" "The output should be empty"
+}
+
+@test "Force remove all remaining containers (only 1 should be left)" {
+  run_toolbox rm --force --all
+  is "$output" "" "The output should be empty"
+}

test/system/302-rmi.bats

@@ -0,0 +1,8 @@
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Remove all images (2 should be present; --force should not be necessary)" {
+  run_toolbox rmi --all
+  is "$output" "" "The output should be empty"
+}

test/system/README.md

@@ -9,36 +9,23 @@ throughout updates of both Toolbox and Podman/libpod.
 
 ## Structure
 
-- **Basic Tests**
-  - [] output version number (Toolbox + Podman)
-  - [x] show help screen when no command is given
-  - [x] create the default container
-  - [x] create a container with a custom name
-  - [x] create a container from a custom image
-  - [x] list containers (no present)
-  - [x] list default container and default image
-  - [x] list containers (some present; different name patterns)
-  - [x] list images (no present)
-  - [x] list images (some present; different name patterns)
-  - [x] remove a specific container
-  - [x] try to remove nonexistent container
-  - [x] try to remove a running container
-  - [x] remove all containers
-  - [x] try to remove all containers (running)
-  - [x] remove a specific image
-  - [x] remove all images
-  - [x] run a command inside of an existing container
-
-- **Advanced Tests**
-  - [ ] create several containers with various configuration and then list them
-  - [ ] create several containers and hop between them (series of enter/exit)
-  - [ ] create a container, enter it, run a series of basic commands (id,
-        whoami, dnf, top, systemctl,..)
-  - [ ] enter a container and test basic set of networking tools (ping,
-        traceroute,..)
-
-The list of tests is stil rather basic. We **welcome** PRs with test
-suggestions or even their implementation.
+- **0xx (Info)**
+  - Commands that are not dependent on the presence/number of containers or
+    images. eg., version, help, etc..
+- **1xx (Initialization)**
+  - Commands (list, create) when Toolbox has not really been used, yet.
+  - It tries to list an empty list, creates several containers (default one
+    and several with custom names and images).
+- **2xx (Usage)**
+  - The created containers are used for the first time testing the
+    initialization (CMD of the container).
+  - Not all containers will be used because in the *Cleanup* phase we want to
+    try removing containers in both running and not running states.
+- **3xx (Cleanup)**
+  - In this section the containers and images from the previous *phases* are
+    removed.
+  - There is a difference between removing running and not running containers.
+    We need to check the right behaviour.
 
 ## Convention
 
@@ -48,6 +35,11 @@ suggestions or even their implementation.
 
 Make sure you have `bats` and `podman` with `toolbox` installed on your system.
 
+**Important**
+Before you start the tests, you need to have present two images: the default
+`fedora-toolbox` image for your version of Fedora and the `fedora-toolbox:29`
+image.
+
 - Enter the toolbox root folder
 - Invoke command `bats ./test/system/` and the test suite should fire up
 

test/system/helpers.bash

@@ -17,40 +17,6 @@ LGC='\033[1;32m' # Light Green Color
 LBC='\033[1;34m' # Light Blue Color
 NC='\033[0m' # No Color
 
-# Basic setup
-function basic_setup() {
-    echo "# [basic_setup]" >&2
-    # Make sure desired images are present
-    if [ -z "$found_needed_image" ]; then
-        run_podman pull "$TOOLBOX_DEFAULT_IMAGE"
-    fi
-}
-
-function setup_with_one_container() {
-    echo "# [setup_with_one_container]" >&2
-    # Clean up all images except for the default one
-    remove_all_images_but_default
-    # Create a new (default) container if no other are present
-    run_toolbox -y create
-}
-
-function basic_teardown() {
-    echo "# [basic_teardown]" >&2
-    # Clean up all containers
-    remove_all_containers
-    # Clean up all images except for the default one
-    remove_all_images_but_default
-}
-
-# Set the default setup function
-function setup() {
-    basic_setup
-}
-
-function teardown() {
-    basic_teardown
-}
-
 
 ################
 #  run_podman  #  Invoke $PODMAN, with timeout, using BATS 'run'