chore: setup integration test

Signed-off-by: Manfred Touron <[email protected]>

.github/workflows/go.yml

@@ -171,3 +171,28 @@ jobs:
           env_vars: OS,GOLANG
           name: codecov-umbrella
           fail_ci_if_error: false
+  integration-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        golang:
+          - 1.15.1
+    env:
+      OS: ubuntu-latest
+      GOLANG: ${{ matrix.golang }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.golang }}
+      - uses: actions/cache@v1
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ matrix.golang }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-${{ matrix.golang }}-
+      - name: Compile the project
+        run: make go.install
+      - name: Run integration tests
+        run: make integration

Makefile

@@ -4,3 +4,6 @@ GOBINS ?=	.
 NPM_PACKAGES ?=	.
 
 include rules.mk
+
+integration: install
+	cd examples && make integration

examples/Makefile

@@ -0,0 +1,14 @@
+integration:
+	# test with testman
+	testman test -run ^TestStable ./...
+	@# FIXME: test unstable tests
+	@# FIXME: test broken tests
+
+	# test with default tools
+	go install moul.io/retry
+	go test -run ^TestStable -count=20 ./... >/dev/null                                                     # should always work
+	retry -m=5  --interval=0 -- "(go test -run ^TestBroken -count=1 ./... >/dev/null)" && exit 1 || exit 0  # should always fail
+	retry -m=50 --interval=0 -- "(go test -run ^TestUnstable -count 1 ./... >/dev/null)"                    # should work at least 1/50
+	go mod tidy
+
+	@echo "SUCCESS."

examples/testpkg/testpkg.go

@@ -0,0 +1,21 @@
+package testpkg
+
+import (
+	"fmt"
+	"math/rand"
+)
+
+func AlwaysSucceed() error {
+	return nil
+}
+
+func AlwaysFailing() error {
+	return fmt.Errorf("hope is the key to life")
+}
+
+func MaySucceed() error {
+	if rand.Intn(3) != 0 {
+		return fmt.Errorf("oops, no luck, try again")
+	}
+	return nil
+}

examples/testpkg/testpkg_test.go

@@ -0,0 +1,33 @@
+package testpkg
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"moul.io/srand"
+)
+
+func ExampleAlwaysSucceed() {
+	fmt.Println(AlwaysSucceed())
+	// Output: <nil>
+}
+
+func TestStableAlwaysSucceed(t *testing.T) {
+	if err := AlwaysSucceed(); err != nil {
+		t.Errorf("expect no error, got %v", err)
+	}
+}
+
+func TestUnstableMaySucceed(t *testing.T) {
+	rand.Seed(srand.Fast())
+	if err := MaySucceed(); err != nil {
+		t.Errorf("expect no error, got %v", err)
+	}
+}
+
+func TestBrokenAlwaysFailing(t *testing.T) {
+	if err := AlwaysFailing(); err != nil {
+		t.Errorf("expect no error, got %v", err)
+	}
+}