Remove msc build tags. Add msc packages. (#666)

* Remove msc build tags. Add msc packages.

This adds more parallelism as a side-effect.

* Remove msc2836 as known broken

.github/workflows/ci.yaml

@@ -41,12 +41,14 @@ jobs:
       matrix:
         include:
           - homeserver: Synapse
-            tags: synapse_blacklist msc3083 msc3787 msc3874 faster_joins
+            tags: synapse_blacklist
+            packages: ./tests/msc3874 ./tests/msc3902
             env: "COMPLEMENT_SHARE_ENV_PREFIX=PASS_ PASS_SYNAPSE_COMPLEMENT_DATABASE=sqlite"
             timeout: 20m
 
           - homeserver: Dendrite
             tags: dendrite_blacklist
+            packages: ""
             env: ""
             timeout: 10m
 
@@ -113,7 +115,7 @@ jobs:
 
       - run: |
           set -o pipefail &&
-          ${{ matrix.env }} go test -v -json -tags "${{ matrix.tags }}" -timeout "${{ matrix.timeout }}" ./tests/... | .ci/scripts/gotestfmt
+          ${{ matrix.env }} go test -v -json -tags "${{ matrix.tags }}" -timeout "${{ matrix.timeout }}" ./tests ./tests/csapi ${{ matrix.packages }} | .ci/scripts/gotestfmt
         shell: bash # required for pipefail to be A Thing. pipefail is required to stop gotestfmt swallowing non-zero exit codes
         name: Run Complement Tests
         env:

helpers/test_main.go

@@ -0,0 +1,41 @@
+package helpers
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/matrix-org/complement/b"
+	"github.com/matrix-org/complement/internal/docker"
+)
+
+var testPackage *TestPackage
+
+// TestMain is the main entry point for Complement.
+//
+// It will clean up any old containers/images/networks from the previous run, then run the tests, then clean up
+// again. No blueprints are made at this point as they are lazily made on demand.
+//
+// The 'namespace' should be unique for this test package, among all test packages which may run in parallel, to avoid
+// docker containers stepping on each other. For MSCs, use the MSC name. For versioned releases, use the version number
+// along with any sub-directory name.
+func TestMain(m *testing.M, namespace string) {
+	var err error
+	testPackage, err = NewTestPackage(namespace)
+	if err != nil {
+		fmt.Printf("Error: %s", err)
+		os.Exit(1)
+	}
+	exitCode := m.Run()
+	testPackage.Cleanup()
+	os.Exit(exitCode)
+}
+
+// Deploy will deploy the given blueprint or terminate the test.
+// It will construct the blueprint if it doesn't already exist in the docker image cache.
+// This function is the main setup function for all tests as it provides a deployment with
+// which tests can interact with.
+func Deploy(t *testing.T, blueprint b.Blueprint) *docker.Deployment {
+	t.Helper()
+	return testPackage.Deploy(t, blueprint)
+}

tests/msc2836/main_test.go

@@ -0,0 +1,18 @@
+package tests
+
+import (
+	"testing"
+
+	"github.com/matrix-org/complement/b"
+	"github.com/matrix-org/complement/helpers"
+	"github.com/matrix-org/complement/internal/docker"
+)
+
+func TestMain(m *testing.M) {
+	helpers.TestMain(m, "msc2836")
+}
+
+func Deploy(t *testing.T, blueprint b.Blueprint) *docker.Deployment {
+	t.Helper()
+	return helpers.Deploy(t, blueprint)
+}

tests/msc2836_test.go → tests/msc2836/msc2836_test.go

@@ -1,6 +1,3 @@
-//go:build msc2836
-// +build msc2836
-
 package tests
 
 import (

tests/msc3391/main_test.go

@@ -0,0 +1,18 @@
+package tests
+
+import (
+	"testing"
+
+	"github.com/matrix-org/complement/b"
+	"github.com/matrix-org/complement/helpers"
+	"github.com/matrix-org/complement/internal/docker"
+)
+
+func TestMain(m *testing.M) {
+	helpers.TestMain(m, "msc3391")
+}
+
+func Deploy(t *testing.T, blueprint b.Blueprint) *docker.Deployment {
+	t.Helper()
+	return helpers.Deploy(t, blueprint)
+}

tests/msc3391_test.go → tests/msc3391/msc3391_test.go

@@ -1,6 +1,3 @@
-//go:build msc3391
-// +build msc3391
-
 // This file contains tests for deleting account data as
 // defined by MSC3391, which you can read here:
 // https://github.com/matrix-org/matrix-doc/pull/3391

tests/msc3874/main_test.go

@@ -0,0 +1,18 @@
+package tests
+
+import (
+	"testing"
+
+	"github.com/matrix-org/complement/b"
+	"github.com/matrix-org/complement/helpers"
+	"github.com/matrix-org/complement/internal/docker"
+)
+
+func TestMain(m *testing.M) {
+	helpers.TestMain(m, "msc3874")
+}
+
+func Deploy(t *testing.T, blueprint b.Blueprint) *docker.Deployment {
+	t.Helper()
+	return helpers.Deploy(t, blueprint)
+}

tests/csapi/room_messages_relation_filter_test.go → tests/msc3874/room_messages_relation_filter_test.go

@@ -1,7 +1,4 @@
-//go:build msc3874
-// +build msc3874
-
-package csapi_tests
+package tests
 
 import (
 	"net/http"
@@ -10,8 +7,8 @@ import (
 
 	"github.com/tidwall/gjson"
 
-	"github.com/matrix-org/complement/client"
 	"github.com/matrix-org/complement/b"
+	"github.com/matrix-org/complement/client"
 	"github.com/matrix-org/complement/match"
 	"github.com/matrix-org/complement/must"
 	"github.com/matrix-org/complement/runtime"

tests/msc3890/main_test.go

@@ -0,0 +1,18 @@
+package tests
+
+import (
+	"testing"
+
+	"github.com/matrix-org/complement/b"
+	"github.com/matrix-org/complement/helpers"
+	"github.com/matrix-org/complement/internal/docker"
+)
+
+func TestMain(m *testing.M) {
+	helpers.TestMain(m, "msc3890")
+}
+
+func Deploy(t *testing.T, blueprint b.Blueprint) *docker.Deployment {
+	t.Helper()
+	return helpers.Deploy(t, blueprint)
+}

tests/msc3890_test.go → tests/msc3890/msc3890_test.go

@@ -1,6 +1,3 @@
-//go:build msc3890
-// +build msc3890
-
 // This file contains tests for local notification settings as
 // defined by MSC3890, which you can read here:
 // https://github.com/matrix-org/matrix-doc/pull/3890

tests/federation_room_join_partial_state_test.go → tests/msc3902/federation_room_join_partial_state_test.go

@@ -1,8 +1,5 @@
-//go:build faster_joins
-// +build faster_joins
-
 // This file contains tests for joining rooms over federation, with the
-// features introduced in msc2775.
+// features introduced in msc3902.
 
 package tests
 

tests/msc3902/main_test.go

@@ -0,0 +1,18 @@
+package tests
+
+import (
+	"testing"
+
+	"github.com/matrix-org/complement/b"
+	"github.com/matrix-org/complement/helpers"
+	"github.com/matrix-org/complement/internal/docker"
+)
+
+func TestMain(m *testing.M) {
+	helpers.TestMain(m, "msc3902")
+}
+
+func Deploy(t *testing.T, blueprint b.Blueprint) *docker.Deployment {
+	t.Helper()
+	return helpers.Deploy(t, blueprint)
+}

tests/msc3930/main_test.go

@@ -0,0 +1,18 @@
+package tests
+
+import (
+	"testing"
+
+	"github.com/matrix-org/complement/b"
+	"github.com/matrix-org/complement/helpers"
+	"github.com/matrix-org/complement/internal/docker"
+)
+
+func TestMain(m *testing.M) {
+	helpers.TestMain(m, "msc3930")
+}
+
+func Deploy(t *testing.T, blueprint b.Blueprint) *docker.Deployment {
+	t.Helper()
+	return helpers.Deploy(t, blueprint)
+}

tests/msc3930_test.go → tests/msc3930/msc3930_test.go

@@ -1,6 +1,3 @@
-//go:build msc3930
-// +build msc3930
-
 // This file contains tests for "push rules of polls" as defined by MSC3930.
 // The MSC that defines the design of the polls system is MSC3381.
 //