Merge pull request #1506 from mdelapenya/testcontainers-go-clickhouse

feat: use testcontainers-go for clickhouse tests

.github/workflows/benchmark.yml

@@ -86,7 +86,7 @@ jobs:
         uses: actions/setup-go@v5
         with:
           # NOTE: Keep this in sync with the version from go.mod
-          go-version: "1.20.x"
+          go-version: "1.21.x"
 
       - name: Setup Node.js
         uses: actions/setup-node@v4
@@ -134,11 +134,6 @@ jobs:
           docker run --name scylladb -p 9042:9042 -p 19042:19042 -p 9160:9160 -p 7000:7000 -p 7001:7001 -p 7199:7199 -p 9180:9180 -d scylladb/scylla:latest --broadcast-address 127.0.0.1 --listen-address 0.0.0.0 --broadcast-rpc-address 127.0.0.1
           sleep 15 # Wait for ScyllaDb to initialize
 
-      - name: Startup Clickhouse
-        run: |
-            docker run -d -p 9001:9000 --name clickhouse --ulimit nofile=262144:262144 clickhouse/clickhouse-server
-            sleep 10 # Wait for Clickhouse to initialize
-
       - name: Setup Redis
         uses: shogo82148/actions-setup-redis@v1
         with:
@@ -178,6 +173,7 @@ jobs:
           POSTGRES_DATABASE: fiber
           POSTGRES_USERNAME: username
           POSTGRES_PASSWORD: "pass#w%rd"
+          TEST_CLICKHOUSE_IMAGE: "clickhouse/clickhouse-server:23-alpine"
 
       - name: Get Previous Benchmark Results
         uses: actions/cache@v4

.github/workflows/test-clickhouse.yml

@@ -20,13 +20,11 @@ jobs:
     steps:
       - name: Fetch Repository
         uses: actions/checkout@v4
-      - name: Startup Clickhouse
-        run: |
-          docker run -d -p 9001:9000 --name clickhouse --ulimit nofile=262144:262144 clickhouse/clickhouse-server
-          sleep 30
       - name: Install Go
         uses: actions/setup-go@v5
         with:
           go-version: '${{ matrix.go-version }}'
       - name: Run Test
+        env:
+          TEST_CLICKHOUSE_IMAGE: clickhouse/clickhouse-server:23-alpine
         run: cd ./clickhouse && go clean -testcache && go test ./... -v -race

clickhouse/README.md

@@ -31,7 +31,13 @@ Install the clickhouse implementation:
 go get github.com/gofiber/storage/clickhouse
 ```
 
-Before running or testing this implementation, you must ensure a Clickhouse cluster is available.
+### Running the tests
+
+This module uses [Testcontainers for Go](https://github.com/testcontainers/testcontainers-go/) to run integration tests, which will start a local instance of Clickhouse as a Docker container under the hood. To run the tests, you must have Docker (or another container runtime 100% compatible with the Docker APIs) installed on your machine.
+
+### Local development
+
+Before running this implementation, you must ensure a Clickhouse cluster is available.
 For local development, we recommend using the Clickhouse Docker image; it contains everything
 necessary for the client to operate correctly.
 

clickhouse/clickhouse_test.go

@@ -1,11 +1,26 @@
 package clickhouse
 
 import (
+	"context"
+	"os"
+	"strconv"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/testcontainers/testcontainers-go/modules/clickhouse"
+)
+
+const (
+	// clickhouseImage is the default image used for running clickhouse in tests.
+	clickhouseImage              = "clickhouse/clickhouse-server:23-alpine"
+	clickhouseImageEnvVar string = "TEST_CLICKHOUSE_IMAGE"
+	clickhouseUser        string = "default"
+	clickhousePass        string = "password"
+	clickhouseDB          string = "fiber"
 )
 
 type TestOrBench interface {
@@ -15,15 +30,48 @@ type TestOrBench interface {
 func getTestConnection(t TestOrBench, cfg Config) (*Storage, error) {
 	t.Helper()
 
+	img := clickhouseImage
+	if imgFromEnv := os.Getenv("TEST_CLICKHOUSE_IMAGE"); imgFromEnv != "" {
+		img = imgFromEnv
+	}
+
+	ctx := context.Background()
+
+	c, err := clickhouse.Run(ctx,
+		img,
+		clickhouse.WithUsername(clickhouseUser),
+		clickhouse.WithPassword(clickhousePass),
+		clickhouse.WithDatabase(clickhouseDB),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	hostPort, err := c.ConnectionHost(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	pair := strings.Split(hostPort, ":")
+	port, err := strconv.Atoi(pair[1])
+	if err != nil {
+		return nil, err
+	}
+
+	// configure the client for the testcontainers clickhouse instance
+	cfg.Host = pair[0]
+	cfg.Port = port
+	cfg.Username = clickhouseUser
+	cfg.Password = clickhousePass
+	cfg.Database = clickhouseDB
+
 	client, err := New(cfg)
 
 	return client, err
 }
 
 func Test_Connection(t *testing.T) {
 	_, err := getTestConnection(t, Config{
-		Host:   "127.0.0.1",
-		Port:   9001,
 		Engine: Memory,
 		Table:  "test_table",
 		Clean:  true,
@@ -34,8 +82,6 @@ func Test_Connection(t *testing.T) {
 
 func Test_Set(t *testing.T) {
 	client, err := getTestConnection(t, Config{
-		Host:   "127.0.0.1",
-		Port:   9001,
 		Engine: Memory,
 		Table:  "test_table",
 		Clean:  true,
@@ -49,8 +95,6 @@ func Test_Set(t *testing.T) {
 
 func Test_Set_With_Exp(t *testing.T) {
 	client, err := getTestConnection(t, Config{
-		Host:   "127.0.0.1",
-		Port:   9001,
 		Engine: Memory,
 		Table:  "test_table",
 		Clean:  true,
@@ -64,8 +108,6 @@ func Test_Set_With_Exp(t *testing.T) {
 
 func Test_Get(t *testing.T) {
 	client, err := getTestConnection(t, Config{
-		Host:   "127.0.0.1",
-		Port:   9001,
 		Engine: Memory,
 		Table:  "test_table",
 		Clean:  true,
@@ -85,8 +127,6 @@ func Test_Get(t *testing.T) {
 
 func Test_Get_With_Exp(t *testing.T) {
 	client, err := getTestConnection(t, Config{
-		Host:   "127.0.0.1",
-		Port:   9001,
 		Engine: Memory,
 		Table:  "test_table",
 		Clean:  true,
@@ -113,8 +153,6 @@ func Test_Get_With_Exp(t *testing.T) {
 
 func Test_Delete(t *testing.T) {
 	client, err := getTestConnection(t, Config{
-		Host:   "127.0.0.1",
-		Port:   9001,
 		Engine: Memory,
 		Table:  "test_table",
 		Clean:  true,
@@ -137,8 +175,6 @@ func Test_Delete(t *testing.T) {
 
 func Test_Reset(t *testing.T) {
 	client, err := getTestConnection(t, Config{
-		Host:   "127.0.0.1",
-		Port:   9001,
 		Engine: Memory,
 		Table:  "test_table",
 		Clean:  true,
@@ -164,8 +200,6 @@ func Benchmark_Clickhouse_Set(b *testing.B) {
 	b.ResetTimer()
 
 	client, err := getTestConnection(b, Config{
-		Host:   "127.0.0.1",
-		Port:   9001,
 		Engine: Memory,
 		Table:  "test_table",
 		Clean:  true,
@@ -186,8 +220,6 @@ func Benchmark_Clickhouse_Get(b *testing.B) {
 	b.ResetTimer()
 
 	client, err := getTestConnection(b, Config{
-		Host:   "127.0.0.1",
-		Port:   9001,
 		Engine: Memory,
 		Table:  "test_table",
 		Clean:  true,
@@ -210,8 +242,6 @@ func Benchmark_Clickhouse_Set_And_Delete(b *testing.B) {
 	b.ResetTimer()
 
 	client, err := getTestConnection(b, Config{
-		Host:   "127.0.0.1",
-		Port:   9001,
 		Engine: Memory,
 		Table:  "test_table",
 		Clean:  true,

clickhouse/go.mod

@@ -5,24 +5,63 @@ go 1.21
 require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.26.0
 	github.com/stretchr/testify v1.9.0
+	github.com/testcontainers/testcontainers-go/modules/clickhouse v0.33.0
 )
 
 require (
+	dario.cat/mergo v1.0.0 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/ClickHouse/ch-go v0.61.5 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/containerd/containerd v1.7.18 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/containerd/platforms v0.2.1 // indirect
+	github.com/cpuguy83/dockercfg v0.3.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/docker v27.1.1+incompatible // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-faster/city v1.0.1 // indirect
 	github.com/go-faster/errors v0.7.1 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/patternmatcher v0.6.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/sys/user v0.1.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/paulmach/orb v0.11.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/testcontainers/testcontainers-go v0.33.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
 	go.opentelemetry.io/otel v1.28.0 // indirect
+	go.opentelemetry.io/otel/metric v1.28.0 // indirect
 	go.opentelemetry.io/otel/trace v1.28.0 // indirect
+	golang.org/x/crypto v0.24.0 // indirect
 	golang.org/x/sys v0.21.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )