Add ability to load triemux from Content Store

Makefile

@@ -29,6 +29,7 @@ unit_tests:
 
 integration_tests: build start_mongo
 	go test -race -v ./integration_tests
+	go test -race -v ./cs_integration_tests
 
 start_mongo:
 	./mongo.sh start

cs_integration_tests/backend_helpers.go

@@ -0,0 +1,99 @@
+package integration
+
+import (
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"time"
+
+	// revive:disable:dot-imports
+	. "github.com/onsi/gomega"
+	// revive:enable:dot-imports
+	"github.com/onsi/gomega/ghttp"
+)
+
+var backends = map[string]string{
+	"backend-1":   "127.0.0.1:6789",
+	"backend-2":   "127.0.0.1:6790",
+	"outer":       "127.0.0.1:6792",
+	"inner":       "127.0.0.1:6793",
+	"innerer":     "127.0.0.1:6794",
+	"root":        "127.0.0.1:6795",
+	"other":       "127.0.0.1:6796",
+	"fallthrough": "127.0.0.1:6797",
+	"down":        "127.0.0.1:6798",
+	"slow-1":      "127.0.0.1:6799",
+	"slow-2":      "127.0.0.1:6800",
+	"backend":     "127.0.0.1:6801",
+	"be":          "127.0.0.1:6802",
+	"not-running": "127.0.0.1:6803",
+	"with-path":   "127.0.0.1:6804",
+}
+
+func startSimpleBackend(identifier, host string) *httptest.Server {
+	l, err := net.Listen("tcp", host)
+	Expect(err).NotTo(HaveOccurred())
+
+	ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, err := w.Write([]byte(identifier))
+		Expect(err).NotTo(HaveOccurred())
+	}))
+	ts.Listener.Close()
+	ts.Listener = l
+	ts.Start()
+	return ts
+}
+
+func startTarpitBackend(host string, delays ...time.Duration) *httptest.Server {
+	responseDelay := 2 * time.Second
+	if len(delays) > 0 {
+		responseDelay = delays[0]
+	}
+	bodyDelay := 0 * time.Second
+	if len(delays) > 1 {
+		bodyDelay = delays[1]
+	}
+
+	l, err := net.Listen("tcp", host)
+	Expect(err).NotTo(HaveOccurred())
+
+	ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body := "Tarpit\n"
+
+		if responseDelay > 0 {
+			time.Sleep(responseDelay)
+		}
+		w.Header().Add("Content-Length", strconv.Itoa(len(body)))
+		w.WriteHeader(http.StatusOK)
+		w.(http.Flusher).Flush()
+
+		if bodyDelay > 0 {
+			time.Sleep(bodyDelay)
+		}
+		_, err := w.Write([]byte(body))
+		Expect(err).NotTo(HaveOccurred())
+	}))
+	ts.Listener.Close()
+	ts.Listener = l
+	ts.Start()
+	return ts
+}
+
+func startRecordingBackend(tls bool, host string) *ghttp.Server {
+	l, err := net.Listen("tcp", host)
+	Expect(err).NotTo(HaveOccurred())
+
+	ts := ghttp.NewUnstartedServer()
+	ts.HTTPTestServer.Listener.Close()
+	ts.HTTPTestServer.Listener = l
+	if tls {
+		ts.HTTPTestServer.StartTLS()
+	} else {
+		ts.Start()
+	}
+
+	ts.AllowUnhandledRequests = true
+	ts.UnhandledRequestStatusCode = http.StatusOK
+	return ts
+}

cs_integration_tests/error_handling_test.go

@@ -0,0 +1,23 @@
+package integration
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("error handling", func() {
+
+	Describe("handling an empty routing table", func() {
+		BeforeEach(func() {
+			reloadRoutes(apiPort)
+		})
+
+		It("should return a 503 error to the client", func() {
+			resp := routerRequest(routerPort, "/")
+			Expect(resp.StatusCode).To(Equal(503))
+
+			resp = routerRequest(routerPort, "/foo")
+			Expect(resp.StatusCode).To(Equal(503))
+		})
+	})
+})

cs_integration_tests/gone_test.go

@@ -0,0 +1,35 @@
+package integration
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Gone routes", func() {
+
+	BeforeEach(func() {
+		addRoute("/foo", NewGoneRoute())
+		addRoute("/bar", NewGoneRoute("prefix"))
+		reloadRoutes(apiPort)
+	})
+
+	It("should support an exact gone route", func() {
+		resp := routerRequest(routerPort, "/foo")
+		Expect(resp.StatusCode).To(Equal(410))
+		Expect(readBody(resp)).To(Equal("410 Gone\n"))
+
+		resp = routerRequest(routerPort, "/foo/bar")
+		Expect(resp.StatusCode).To(Equal(404))
+		Expect(readBody(resp)).To(Equal("404 page not found\n"))
+	})
+
+	It("should support a prefix gone route", func() {
+		resp := routerRequest(routerPort, "/bar")
+		Expect(resp.StatusCode).To(Equal(410))
+		Expect(readBody(resp)).To(Equal("410 Gone\n"))
+
+		resp = routerRequest(routerPort, "/bar/baz")
+		Expect(resp.StatusCode).To(Equal(410))
+		Expect(readBody(resp)).To(Equal("410 Gone\n"))
+	})
+})

cs_integration_tests/http_request_helpers.go

@@ -0,0 +1,83 @@
+package integration
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/textproto"
+
+	// revive:disable:dot-imports
+	. "github.com/onsi/gomega"
+	// revive:enable:dot-imports
+)
+
+func routerRequest(port int, path string) *http.Response {
+	return doRequest(newRequest("GET", routerURL(port, path)))
+}
+
+func routerRequestWithHeaders(port int, path string, headers map[string]string) *http.Response {
+	return doRequest(newRequestWithHeaders("GET", routerURL(port, path), headers))
+}
+
+func newRequest(method, url string) *http.Request {
+	req, err := http.NewRequestWithContext(context.Background(), method, url, nil)
+	Expect(err).NotTo(HaveOccurred())
+	return req
+}
+
+func newRequestWithHeaders(method, url string, headers map[string]string) *http.Request {
+	req := newRequest(method, url)
+	for k, v := range headers {
+		req.Header.Set(k, v)
+	}
+	return req
+}
+
+func doRequest(req *http.Request) *http.Response {
+	if _, ok := req.Header[textproto.CanonicalMIMEHeaderKey("User-Agent")]; !ok {
+		// Setting a blank User-Agent causes the http lib not to output one, whereas if there
+		// is no header, it will output a default one.
+		// See: https://github.com/golang/go/blob/release-branch.go1.5/src/net/http/request.go#L419
+		req.Header.Set("User-Agent", "")
+	}
+	resp, err := http.DefaultTransport.RoundTrip(req)
+	Expect(err).NotTo(HaveOccurred())
+	return resp
+}
+
+func doHTTP10Request(req *http.Request) *http.Response {
+	conn, err := net.Dial("tcp", req.URL.Host)
+	Expect(err).NotTo(HaveOccurred())
+	defer conn.Close()
+
+	if req.Method == "" {
+		req.Method = "GET"
+	}
+	req.Proto = "HTTP/1.0"
+	req.ProtoMinor = 0
+	fmt.Fprintf(conn, "%s %s %s\r\n", req.Method, req.URL.RequestURI(), req.Proto)
+	err = req.Header.Write(conn)
+	Expect(err).NotTo(HaveOccurred())
+	fmt.Fprintf(conn, "\r\n")
+
+	resp, err := http.ReadResponse(bufio.NewReader(conn), req)
+	Expect(err).NotTo(HaveOccurred())
+	return resp
+}
+
+func readBody(resp *http.Response) string {
+	bytes, err := io.ReadAll(resp.Body)
+	Expect(err).NotTo(HaveOccurred())
+	return string(bytes)
+}
+
+func readJSONBody(resp *http.Response, data interface{}) {
+	bytes, err := io.ReadAll(resp.Body)
+	Expect(err).NotTo(HaveOccurred())
+	err = json.Unmarshal(bytes, data)
+	Expect(err).NotTo(HaveOccurred())
+}

cs_integration_tests/integration_test.go

@@ -0,0 +1,59 @@
+package integration
+
+import (
+	"context"
+	"runtime"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var cleanupPostgresContainer func()
+
+func TestEverything(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Integration test suite")
+}
+
+var _ = BeforeSuite(func() {
+	runtime.GOMAXPROCS(runtime.NumCPU())
+	var err error
+	err = setupTempLogfile()
+	if err != nil {
+		Fail(err.Error())
+	}
+
+	ctx := context.Background()
+
+	postgresContainer, cleanupPostgresContainer, err = runPostgresContainer(ctx)
+
+	if err != nil {
+		Fail(err.Error())
+	}
+
+	backendEnvVars := []string{}
+	for id, host := range backends {
+		envVar := "BACKEND_URL_" + id + "=http://" + host
+		backendEnvVars = append(backendEnvVars, envVar)
+	}
+
+	err = startRouter(routerPort, apiPort, backendEnvVars)
+	if err != nil {
+		Fail(err.Error())
+	}
+	err = initRouteHelper()
+	if err != nil {
+		Fail(err.Error())
+	}
+})
+
+var _ = BeforeEach(func() {
+	resetTempLogfile()
+})
+
+var _ = AfterSuite(func() {
+	stopRouter(routerPort)
+	cleanupPostgresContainer()
+	cleanupTempLogfile()
+})

cs_integration_tests/metrics_test.go

@@ -0,0 +1,22 @@
+package integration
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("/metrics API endpoint", func() {
+	Context("response body", func() {
+		var responseBody string
+
+		BeforeEach(func() {
+			resp := doRequest(newRequest("GET", routerURL(apiPort, "/metrics")))
+			Expect(resp.StatusCode).To(Equal(200))
+			responseBody = readBody(resp)
+		})
+
+		It("should contain at least one metric", func() {
+			Expect(responseBody).To(ContainSubstring("router_"))
+		})
+	})
+})