diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 087852ee..a438b703 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -30,6 +30,9 @@ jobs:
         npm -g install newman
         ./testing/run_api_tests.sh
 
+    - name: Mail Tests
+      run: ./testing/run_mail_tests.sh
+
   migration:
     name: Migration tests
     runs-on: ubuntu-latest
diff --git a/services/mail/smtp_test.go b/services/mail/smtp_test.go
index 20eb6cb7..23ec748b 100644
--- a/services/mail/smtp_test.go
+++ b/services/mail/smtp_test.go
@@ -13,15 +13,17 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"net"
+	"net/http"
+	"testing"
+	"time"
+
 	"github.com/emvi/logbuch"
 	"github.com/muety/wakapi/config"
 	"github.com/muety/wakapi/models"
 	"github.com/muety/wakapi/utils"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
-	"net/http"
-	"testing"
-	"time"
 )
 
 const (
@@ -46,10 +48,23 @@ func (suite *SmtpTestSuite) BeforeTest(suiteName, testName string) {
 }
 
 func TestSmtpTestSuite(t *testing.T) {
-	if smtp4dev := newSmtp4DevClient(); smtp4dev.Check() != nil {
-		t.Skip(fmt.Sprintf("WARNING: smtp4Dev not available at %s - skipping smtp tests", smtp4dev.ApiBaseUrl))
+	smtp4dev := newSmtp4DevClient()
+
+	address := net.JoinHostPort(Smtp4DevHost, fmt.Sprintf("%d", Smtp4DevPort))
+	_, err := net.DialTimeout("tcp", address, time.Second)
+	if err != nil {
+		t.Skipf("WARNING: smtp4Dev not available at %s - skipping smtp tests", address)
 		return
 	}
+
+	for i := 0; i < 5; i++ {
+		if smtp4dev.Check() == nil {
+			break
+		}
+		t.Logf("smtp4Dev not ready at %s, retrying...", smtp4dev.ApiBaseUrl)
+		time.Sleep(1 * time.Second)
+	}
+
 	suite.Run(t, new(SmtpTestSuite))
 }
 
@@ -201,21 +216,21 @@ func (c *Smtp4DevClient) CountMessages() (int, error) {
 func (c *Smtp4DevClient) SetNoTls() error {
 	logbuch.Info("[smtp4Dev] disabling tls encryption")
 	err := c.SetConfigValue("tlsMode", "None")
-	time.Sleep(1 * time.Second)
+	time.Sleep(2 * time.Second)
 	return err
 }
 
 func (c *Smtp4DevClient) SetForcedTls() error {
 	logbuch.Info("[smtp4Dev] enabling forced tls encryption")
 	err := c.SetConfigValue("tlsMode", "ImplicitTls")
-	time.Sleep(1 * time.Second)
+	time.Sleep(2 * time.Second)
 	return err
 }
 
 func (c *Smtp4DevClient) SetStartTls() error {
 	logbuch.Info("[smtp4Dev] enabling tls encryption via starttls")
 	err := c.SetConfigValue("tlsMode", "StartTls")
-	time.Sleep(1 * time.Second)
+	time.Sleep(2 * time.Second)
 	return err
 }
 
diff --git a/testing/run_api_tests.sh b/testing/run_api_tests.sh
index 6e6211fe..82816908 100755
--- a/testing/run_api_tests.sh
+++ b/testing/run_api_tests.sh
@@ -4,7 +4,7 @@ set -o nounset -o pipefail -o errexit
 DB_TYPE=${1-sqlite}
 
 if ! command -v newman &> /dev/null; then
-    echo "Newman could not be found. Run 'npm install -g newman' first."
+    echo "newman could not be found. Run 'npm install -g newman' first."
     exit 1
 fi
 
@@ -135,7 +135,6 @@ start_wakapi_background "../wakapi" "$config"
 kill_wakapi
 
 # Only sqlite has data
-
 if [ "$DB_TYPE" == "sqlite" ]; then
     echo "Creating database and schema ..."
     sqlite3 wakapi_testing.db < schema.sql
@@ -145,10 +144,9 @@ if [ "$DB_TYPE" == "sqlite" ]; then
     start_wakapi_background "../wakapi" "$config"
     echo "Running test collection ..."
     if ! newman run "wakapi_api_tests.postman_collection.json"; then
-        exit_code=$?
-        echo "newman failed with exit code $exit_code"
-        exit "$exit_code"
+        echo "newman failed"
+        exit 1
     fi
+
     kill_wakapi
 fi
-
diff --git a/testing/run_mail_tests.sh b/testing/run_mail_tests.sh
old mode 100644
new mode 100755
index 029dbd29..d51402ed
--- a/testing/run_mail_tests.sh
+++ b/testing/run_mail_tests.sh
@@ -1,10 +1,18 @@
 #!/bin/bash
+set -o nounset -o errexit -o pipefail
+
+cleanup() {
+    echo "Stopping and removing smtpdev ..."
+    docker stop smtp4dev_wakapi &> /dev/null || true
+    docker rm -f smtp4dev_wakapi &> /dev/null
+}
+trap cleanup EXIT
+
+cleanup
 
 echo "Starting smtp4dev in Docker ..."
 docker run -d --rm -p 2525:25 -p 8080:80 --name smtp4dev_wakapi rnwood/smtp4dev
 
 echo "Running tests ..."
-go test -run TestSmtpTestSuite ../services/mail
-
-echo "Killing container again ..."
-docker stop smtp4dev_wakapi
\ No newline at end of file
+script_dir=$(dirname "${BASH_SOURCE[0]}")
+go test -count=1 -run TestSmtpTestSuite "$script_dir/../services/mail"