From f9a22ee538c637ab57203e1f73ca9a5a113c54bd Mon Sep 17 00:00:00 2001 From: Lucas Lemos Date: Tue, 9 Apr 2024 13:52:50 -0300 Subject: [PATCH 01/34] feat: add clickhouse implementation --- .github/workflows/test-clickhouse.yml | 33 ++++++++ clickhouse/README.md | 114 ++++++++++++++++++++++++++ clickhouse/clickhouse.go | 113 +++++++++++++++++++++++++ clickhouse/clickhouse_test.go | 110 +++++++++++++++++++++++++ clickhouse/config.go | 113 +++++++++++++++++++++++++ clickhouse/go.mod | 28 +++++++ clickhouse/go.sum | 110 +++++++++++++++++++++++++ go.mod | 4 +- 8 files changed, 624 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test-clickhouse.yml create mode 100644 clickhouse/README.md create mode 100644 clickhouse/clickhouse.go create mode 100644 clickhouse/clickhouse_test.go create mode 100644 clickhouse/config.go create mode 100644 clickhouse/go.mod create mode 100644 clickhouse/go.sum diff --git a/.github/workflows/test-clickhouse.yml b/.github/workflows/test-clickhouse.yml new file mode 100644 index 000000000..1e2c8b3d4 --- /dev/null +++ b/.github/workflows/test-clickhouse.yml @@ -0,0 +1,33 @@ +on: + push: + branches: + - master + - main + paths: + - 'clickhouse/**' + pull_request: + paths: + - 'clickhouse/**' +name: 'Tests Clickhouse' +jobs: + Tests: + runs-on: ubuntu-latest + strategy: + matrix: + go-version: + - 1.19.x + - 1.20.x + - 1.21.x + steps: + - name: Fetch Repository + uses: actions/checkout@v4 + - name: Startup Clickhouse + run: | + docker run -d -p 9000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server + sleep 30 + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: '${{ matrix.go-version }}' + - name: Run Test + run: cd ./clickhouse && go clean -testcache && go test ./... -v diff --git a/clickhouse/README.md b/clickhouse/README.md new file mode 100644 index 000000000..e9893f081 --- /dev/null +++ b/clickhouse/README.md @@ -0,0 +1,114 @@ +# Clickhouse +A Clickhouse storage driver using [https://github.com/ClickHouse/clickhouse-go](https://github.com/ClickHouse/clickhouse-go). + +### Table of Contents +- [Signatures](#signatures) +- [Installation](#installation) +- [Examples](#examples) +- [Config](#config) +- [Default Config](#default-config) + +### Signatures +```go +func New(config ...Config) (*Storage, error) +func (s *Storage) Get(key string) ([]byte, error) +func (s *Storage) Set(key string, val []byte, exp time.Duration) error +func (s *Storage) Delete(key string) error +func (s *Storage) Reset() error +func (s *Storage) Close() error +func (s *Storage) Conn() *Session +``` + +### Installation +Clickhouse is supported on Go versions 1.19 and above: + +Install the clickhouse implementation: +```bash +go get github.com/gofiber/storage/clickhouse +``` + +Before running or testing this implementation, you must ensure a Clickhouse cluster is available. +For local development, we recommend using the Clickhouse Docker image; it contains everything +necessary for the client to operate correctly. + +To start Clickhouse using Docker, issue the following: + +```bash +docker run -d -p 9000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server +``` + +After running this command you're ready to start using the storage and connecting to the database. + +### Examples +Import the storage package. +```go +import "github.com/gofiber/storage/coherence" +``` + +You can use the following possibilities to create a storage: +```go +// Initialize default config, to connect to localhost:9000 using the memory engine and with a clean table. +store, err := clickhouse.New(clickhouse.Config{ + Host: "localhost", + Port: 9000, + Clean: true, +}) + +// Initialize custom config to connect to a different host/port and use custom engine and with clean table. +store, err := clickhouse.New(clickhouse.Config{ + Host: "some-ip-address", + Port: 9000, + Engine: clickhouse.MergeTree, + Clean: true, +}) + +// Initialize to connect with TLS enabled with your own tls.Config and with clean table. +tlsConfig := config := &tls.Config{...} + +store, err := clickhouse.New(coherence.Config{ + Host: "some-ip-address", + Port: 9000, + Clean: true, + TLSConfig: tlsConfig, +}) +``` + +### Config + +```go +// Config defines configuration options for Clickhouse connection. +type Config struct { + // The host of the database. Ex: 127.0.0.1 + Host string + // The port where the database is supposed to listen to. Ex: 9000 + Port int + // The database that the connection should authenticate from + Database string + // The username to be used in the authentication + Username string + // The password to be used in the authentication + Password string + // The name of the table that will store the data + Table string + // The engine that should be used in the table + Engine string + // Should start a clean table, default false + Clean bool + // TLS configuration, default nil + TLSConfig *tls.Config + // Should the connection be in debug mode, default false + Debug bool + // The function to use with the debug config, default print function. It only works when debug is true + Debugf func(format string, v ...any) +} +``` + +### Default Config +```go +var DefaultConfig = Config{ + Host: "localhost", + Port: 9000, + Engine: "Memory", + Clean: false, +} +``` \ No newline at end of file diff --git a/clickhouse/clickhouse.go b/clickhouse/clickhouse.go new file mode 100644 index 000000000..8f44abfbb --- /dev/null +++ b/clickhouse/clickhouse.go @@ -0,0 +1,113 @@ +package clickhouse + +import ( + "context" + "fmt" + "time" + + driver "github.com/ClickHouse/clickhouse-go/v2" +) + +type Storage struct { + session driver.Conn + context context.Context + table string +} + +// New returns a new [*Storage] given a [Config]. +func New(configuration Config) (*Storage, error) { + cfg, engine, err := defaultConfig(configuration) + if err != nil { + return &Storage{}, err + } + + conn, err := driver.Open(&cfg) + if err != nil { + return &Storage{}, err + } + + ctx := driver.Context(context.Background(), driver.WithParameters(driver.Parameters{ + "table": configuration.Table, + })) + + if configuration.Clean { + queryWithEngine := fmt.Sprintf(createTableString, engine) + if err := conn.Exec(ctx, queryWithEngine); err != nil { + return &Storage{}, err + } + + if err := conn.Exec(ctx, resetDataString); err != nil { + return &Storage{}, err + } + } + + return &Storage{ + session: conn, + context: ctx, + table: configuration.Table, + }, conn.Ping(ctx) +} + +func (s *Storage) Set(key string, value string, expiration time.Duration) error { + if len(key) <= 0 || len(value) <= 0 { + return nil + } + + exp := time.Time{} + if expiration != 0 { + exp = time.Now().Add(expiration).UTC() + } + + ctx := driver.Context(s.context, driver.WithParameters(driver.Parameters{ + "key": key, + "value": value, + "expiration": exp.Format("2006-01-02 15:04:05"), + "table": s.table, + })) + + err := s. + session. + Exec( + ctx, + insertDataString, + ) + + return err +} + +func (s *Storage) Get(key string) ([]byte, error) { + var resultSlice []schema + + ctx := driver.Context(s.context, driver.WithParameters(driver.Parameters{ + "key": key, + "table": s.table, + })) + err := s.session.Select(ctx, &resultSlice, selectDataString) + if err != nil { + return nil, err + } + result := resultSlice[0] + + if !result.Expiration.IsZero() && result.Expiration.UTC().Unix() <= time.Now().UTC().Unix() { + return nil, nil + } + + return []byte(result.Value), err +} + +func (s *Storage) Delete(key string) error { + ctx := driver.Context(s.context, driver.WithParameters(driver.Parameters{ + "table": s.table, + "key": key, + })) + + return s.session.Exec(ctx, deleteDataString) +} + +func (s *Storage) Reset() error { + return s.session.Exec(s.context, resetDataString) +} + +func (s *Storage) Close() error { + return s.session.Close() +} diff --git a/clickhouse/clickhouse_test.go b/clickhouse/clickhouse_test.go new file mode 100644 index 000000000..6a27d2a8e --- /dev/null +++ b/clickhouse/clickhouse_test.go @@ -0,0 +1,110 @@ +package clickhouse + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func getTestConnection(t *testing.T) (*Storage, error) { + t.Helper() + + client, err := New(Config{ + Host: "127.0.0.1", + Port: 9000, + Engine: "Memory", + Table: "test_table", + Clean: true, + }) + + return client, err +} + +func TestConnection(t *testing.T) { + _, err := getTestConnection(t) + + require.NoError(t, err) +} + +func TestSet(t *testing.T) { + client, err := getTestConnection(t) + require.NoError(t, err) + defer client.Close() + + err = client.Set("somekey", "somevalue", 0) + require.NoError(t, err) +} + +func TestSetWithExp(t *testing.T) { + client, err := getTestConnection(t) + require.NoError(t, err) + defer client.Close() + + err = client.Set("setsomekeywithexp", "somevalue", time.Second*1) + require.NoError(t, err) +} + +func TestGet(t *testing.T) { + client, err := getTestConnection(t) + require.NoError(t, err) + defer client.Close() + + err = client.Set("somekey", "somevalue", 0) + require.NoError(t, err) + + value, err := client.Get("somekey") + + require.NoError(t, err) + assert.NotNil(t, value) + assert.Equal(t, "somevalue", string(value)) +} + +func TestGetWithExp(t *testing.T) { + client, err := getTestConnection(t) + require.NoError(t, err) + defer client.Close() + + err = client.Set("getsomekeywithexp", "somevalue", time.Second*5) + require.NoError(t, err) + + value, err := client.Get("getsomekeywithexp") + + require.NoError(t, err) + assert.NotNil(t, value) + assert.Equal(t, "somevalue", string(value)) + + time.Sleep(time.Second * 10) + + value, err = client.Get("getsomekeywithexp") + + require.NoError(t, err) + assert.Nil(t, value) +} + +func TestDelete(t *testing.T) { + client, err := getTestConnection(t) + require.NoError(t, err) + defer client.Close() + + err = client.Set("somekeytodelete", "somevalue", time.Second*5) + require.NoError(t, err) + + err = client.Delete("somekeytodelete") + + require.NoError(t, err) +} + +func TestReset(t *testing.T) { + client, err := getTestConnection(t) + require.NoError(t, err) + defer client.Close() + + err = client.Set("testkey", "somevalue", 0) + require.NoError(t, err) + + err = client.Reset() + + require.NoError(t, err) +} diff --git a/clickhouse/config.go b/clickhouse/config.go new file mode 100644 index 000000000..f4b27dca0 --- /dev/null +++ b/clickhouse/config.go @@ -0,0 +1,113 @@ +package clickhouse + +import ( + "crypto/tls" + "errors" + "fmt" + "time" + + driver "github.com/ClickHouse/clickhouse-go/v2" +) + +type schema struct { + Value string `ch:"value"` + Expiration time.Time `ch:"expiration"` +} + +const ( + Memory = "Memory" + MergeTree = "MergeTree" + StripeLog = "StripeLog" + TinyLog = "TinyLog" + Log = "Log" +) + +// Config defines configuration options for Clickhouse connection. +type Config struct { + // The host of the database. Ex: 127.0.0.1 + Host string + // The port where the database is supposed to listen to. Ex: 9000 + Port int + // The database that the connection should authenticate from + Database string + // The username to be used in the authentication + Username string + // The password to be used in the authentication + Password string + // The name of the table that will store the data + Table string + // The engine that should be used in the table + Engine string + // Should start a clean table, default false + Clean bool + // TLS configuration, default nil + TLSConfig *tls.Config + // Should the connection be in debug mode, default false + Debug bool + // The function to use with the debug config, default print function. It only works when debug is true + Debugf func(format string, v ...any) +} + +func defaultConfig(configuration Config) (driver.Options, string, error) { + if configuration.Table == "" { + return driver.Options{}, "", errors.New("table name not provided") + } + + if configuration.Host == "" { + configuration.Host = "localhost" + } + + if configuration.Port == 0 { + configuration.Port = 9000 + } + + if configuration.Engine == "" { + configuration.Engine = "Memory" + } + + config := driver.Options{ + Addr: []string{fmt.Sprintf("%s:%d", configuration.Host, configuration.Port)}, + } + + if configuration.Username != "" && configuration.Password != "" { + config.Auth = driver.Auth{ + Database: configuration.Database, + Username: configuration.Username, + Password: configuration.Password, + } + } + + if configuration.TLSConfig != nil { + config.TLS = configuration.TLSConfig + } + + if configuration.Debug && config.Debugf == nil { + config.Debugf = func(format string, v ...any) { fmt.Printf(format, v...) } + } + + return config, configuration.Engine, nil +} + +const resetDataString = ` + TRUNCATE TABLE {table:Identifier} +` + +const deleteDataString = ` + ALTER TABLE {table:Identifier} DELETE WHERE key = {key:String} +` + +const selectDataString = ` + SELECT value, expiration FROM {table:Identifier} WHERE key = {key:String} +` + +const insertDataString = ` + INSERT INTO {table:Identifier} (*) VALUES ({key:String}, {value:String}, {expiration:Datetime}) +` + +const createTableString = ` + CREATE TABLE IF NOT EXISTS {table:Identifier} ( + key String + , value String + , expiration Datetime + ) ENGINE=%s +` diff --git a/clickhouse/go.mod b/clickhouse/go.mod new file mode 100644 index 000000000..42b968aea --- /dev/null +++ b/clickhouse/go.mod @@ -0,0 +1,28 @@ +module github.com/gofiber/storage/clickhouse + +go 1.22.0 + +require ( + github.com/ClickHouse/clickhouse-go/v2 v2.20.0 + github.com/stretchr/testify v1.8.4 +) + +require ( + github.com/ClickHouse/ch-go v0.61.3 // indirect + github.com/andybalholm/brotli v1.1.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.17.7 // indirect + github.com/paulmach/orb v0.11.1 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/segmentio/asm v1.2.0 // indirect + github.com/shopspring/decimal v1.3.1 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + golang.org/x/sys v0.17.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/clickhouse/go.sum b/clickhouse/go.sum new file mode 100644 index 000000000..c8427c637 --- /dev/null +++ b/clickhouse/go.sum @@ -0,0 +1,110 @@ +github.com/ClickHouse/ch-go v0.61.3 h1:MmBwUhXrAOBZK7n/sWBzq6FdIQ01cuF2SaaO8KlDRzI= +github.com/ClickHouse/ch-go v0.61.3/go.mod h1:1PqXjMz/7S1ZUaKvwPA3i35W2bz2mAMFeCi6DIXgGwQ= +github.com/ClickHouse/clickhouse-go/v2 v2.20.0 h1:bvlLQ31XJfl7MxIqAq2l1G6JhHYzqEXdvfpMeU6bkKc= +github.com/ClickHouse/clickhouse-go/v2 v2.20.0/go.mod h1:VQfyA+tCwCRw2G7ogfY8V0fq/r0yJWzy8UDrjiP/Lbs= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= +github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= +github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= +github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go.mod b/go.mod index aa4d08f55..d7b1e1046 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module github.com/gofiber/storage -go 1.19 \ No newline at end of file +go 1.21 + +toolchain go1.22.0 From efea41904392f7d602a66e59d85a2593888e8ae7 Mon Sep 17 00:00:00 2001 From: Lucas Lemos Date: Wed, 10 Apr 2024 14:13:36 -0300 Subject: [PATCH 02/34] fix: invalid go version error --- clickhouse/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse/go.mod b/clickhouse/go.mod index 42b968aea..a98b8b21c 100644 --- a/clickhouse/go.mod +++ b/clickhouse/go.mod @@ -1,6 +1,6 @@ module github.com/gofiber/storage/clickhouse -go 1.22.0 +go 1.22 require ( github.com/ClickHouse/clickhouse-go/v2 v2.20.0 From b8c51d17792ab07518550a93972c7f0498648d24 Mon Sep 17 00:00:00 2001 From: Lucas Lemos Date: Wed, 10 Apr 2024 14:13:51 -0300 Subject: [PATCH 03/34] fix: add benchmark and fix mismatched type on set function --- clickhouse/clickhouse.go | 4 ++-- clickhouse/clickhouse_test.go | 35 ++++++++++++++++++++++++++++------- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/clickhouse/clickhouse.go b/clickhouse/clickhouse.go index 8f44abfbb..e43c79410 100644 --- a/clickhouse/clickhouse.go +++ b/clickhouse/clickhouse.go @@ -48,7 +48,7 @@ func New(configuration Config) (*Storage, error) { }, conn.Ping(ctx) } -func (s *Storage) Set(key string, value string, expiration time.Duration) error { +func (s *Storage) Set(key string, value []byte, expiration time.Duration) error { if len(key) <= 0 || len(value) <= 0 { return nil } @@ -60,7 +60,7 @@ func (s *Storage) Set(key string, value string, expiration time.Duration) error ctx := driver.Context(s.context, driver.WithParameters(driver.Parameters{ "key": key, - "value": value, + "value": string(value), "expiration": exp.Format("2006-01-02 15:04:05"), "table": s.table, })) diff --git a/clickhouse/clickhouse_test.go b/clickhouse/clickhouse_test.go index 6a27d2a8e..235c8b681 100644 --- a/clickhouse/clickhouse_test.go +++ b/clickhouse/clickhouse_test.go @@ -8,7 +8,11 @@ import ( "github.com/stretchr/testify/require" ) -func getTestConnection(t *testing.T) (*Storage, error) { +type TestOrBench interface { + Helper() +} + +func getTestConnection(t TestOrBench) (*Storage, error) { t.Helper() client, err := New(Config{ @@ -33,7 +37,7 @@ func TestSet(t *testing.T) { require.NoError(t, err) defer client.Close() - err = client.Set("somekey", "somevalue", 0) + err = client.Set("somekey", []byte("somevalue"), 0) require.NoError(t, err) } @@ -42,7 +46,7 @@ func TestSetWithExp(t *testing.T) { require.NoError(t, err) defer client.Close() - err = client.Set("setsomekeywithexp", "somevalue", time.Second*1) + err = client.Set("setsomekeywithexp", []byte("somevalue"), time.Second*1) require.NoError(t, err) } @@ -51,7 +55,7 @@ func TestGet(t *testing.T) { require.NoError(t, err) defer client.Close() - err = client.Set("somekey", "somevalue", 0) + err = client.Set("somekey", []byte("somevalue"), 0) require.NoError(t, err) value, err := client.Get("somekey") @@ -66,7 +70,7 @@ func TestGetWithExp(t *testing.T) { require.NoError(t, err) defer client.Close() - err = client.Set("getsomekeywithexp", "somevalue", time.Second*5) + err = client.Set("getsomekeywithexp", []byte("somevalue"), time.Second*5) require.NoError(t, err) value, err := client.Get("getsomekeywithexp") @@ -88,7 +92,7 @@ func TestDelete(t *testing.T) { require.NoError(t, err) defer client.Close() - err = client.Set("somekeytodelete", "somevalue", time.Second*5) + err = client.Set("somekeytodelete", []byte("somevalue"), time.Second*5) require.NoError(t, err) err = client.Delete("somekeytodelete") @@ -101,10 +105,27 @@ func TestReset(t *testing.T) { require.NoError(t, err) defer client.Close() - err = client.Set("testkey", "somevalue", 0) + err = client.Set("testkey", []byte("somevalue"), 0) require.NoError(t, err) err = client.Reset() require.NoError(t, err) } + +func BenchmarkClickhouse(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + client, err := getTestConnection(b) + + require.NoError(b, err) + defer client.Close() + + for i := 0; i < b.N; i++ { + _ = client.Set("john", []byte("doe"), 0) + err = client.Delete("john") + } + + require.NoError(b, err) +} From 2baf7d7e477d2a41db80b6e694d4de3cec4f8039 Mon Sep 17 00:00:00 2001 From: Lucas Lemos Date: Tue, 16 Apr 2024 14:11:18 -0300 Subject: [PATCH 04/34] revert: go.mod golang version change --- go.mod | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d7b1e1046..2d6880fe0 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,3 @@ module github.com/gofiber/storage -go 1.21 - -toolchain go1.22.0 +go 1.19 From de3083f492073de3a0d8655aa655b365706c8a2c Mon Sep 17 00:00:00 2001 From: Lucas Lemos Date: Tue, 16 Apr 2024 14:11:54 -0300 Subject: [PATCH 05/34] chore: add endline to root go.mod --- go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/go.mod b/go.mod index 2d6880fe0..7ec801500 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,4 @@ module github.com/gofiber/storage go 1.19 + From 4c5f2ef1b86c2513b18019b9b7f5634d56a3da52 Mon Sep 17 00:00:00 2001 From: Lucas Lemos Date: Tue, 16 Apr 2024 14:12:24 -0300 Subject: [PATCH 06/34] chore: remove endline from go.mod file --- go.mod | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 7ec801500..aa4d08f55 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,3 @@ module github.com/gofiber/storage -go 1.19 - +go 1.19 \ No newline at end of file From 8a64cac72fe165567822b6f3f78a8ca27ee27439 Mon Sep 17 00:00:00 2001 From: Juan Calderon-Perez <835733+gaby@users.noreply.github.com> Date: Tue, 16 Apr 2024 19:43:13 -0400 Subject: [PATCH 07/34] Update go.mod --- clickhouse/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse/go.mod b/clickhouse/go.mod index a98b8b21c..4b5909378 100644 --- a/clickhouse/go.mod +++ b/clickhouse/go.mod @@ -1,6 +1,6 @@ module github.com/gofiber/storage/clickhouse -go 1.22 +go 1.21 require ( github.com/ClickHouse/clickhouse-go/v2 v2.20.0 From 81b668ae404decfb86c9153eb03a06903a62905a Mon Sep 17 00:00:00 2001 From: Juan Calderon-Perez <835733+gaby@users.noreply.github.com> Date: Tue, 16 Apr 2024 19:44:02 -0400 Subject: [PATCH 08/34] Update test-clickhouse.yml --- .github/workflows/test-clickhouse.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-clickhouse.yml b/.github/workflows/test-clickhouse.yml index 1e2c8b3d4..17e9aab7b 100644 --- a/.github/workflows/test-clickhouse.yml +++ b/.github/workflows/test-clickhouse.yml @@ -15,9 +15,8 @@ jobs: strategy: matrix: go-version: - - 1.19.x - - 1.20.x - 1.21.x + - 1.22.x steps: - name: Fetch Repository uses: actions/checkout@v4 From 99c099431ad846c09686eadc27aa5b77b8872fdb Mon Sep 17 00:00:00 2001 From: Juan Calderon-Perez <835733+gaby@users.noreply.github.com> Date: Tue, 16 Apr 2024 19:48:09 -0400 Subject: [PATCH 09/34] Update test-cloudflarekv.yml --- .github/workflows/test-cloudflarekv.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-cloudflarekv.yml b/.github/workflows/test-cloudflarekv.yml index 4d819ba2d..d22d99fbb 100644 --- a/.github/workflows/test-cloudflarekv.yml +++ b/.github/workflows/test-cloudflarekv.yml @@ -3,10 +3,13 @@ name: Tests CloudflareKV on: push: branches: + - master - main + paths: + - 'clickhouse/**' pull_request: - branches: - - main + paths: + - 'clickhouse/**' jobs: Tests: @@ -16,7 +19,6 @@ jobs: go-version: - 1.21.x - 1.22.x - steps: - name: Checkout Repository uses: actions/checkout@v4 From 21ac41f9613f09d4dfd740e9c3b89504f93c08a2 Mon Sep 17 00:00:00 2001 From: Lucas Lemos Date: Thu, 18 Apr 2024 15:20:41 -0300 Subject: [PATCH 10/34] ci: add clickhosue to benchmarks.yml --- .github/workflows/benchmark.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e7323812c..cf3a51483 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -134,6 +134,11 @@ jobs: docker run --name scylladb -p 9042:9042 -p 19042:19042 -p 9160:9160 -p 7000:7000 -p 7001:7001 -p 7199:7199 -p 9180:9180 -d scylladb/scylla:latest --broadcast-address 127.0.0.1 --listen-address 0.0.0.0 --broadcast-rpc-address 127.0.0.1 sleep 15 # Wait for ScyllaDb to initialize + - name: Startup Clickhouse + run: | + docker run -d -p 9000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server + sleep 10 # Wait for Clickhouse to initialize + - name: Setup Redis uses: shogo82148/actions-setup-redis@v1 with: From 3b83def333ef331136381fe2b75e749b3dabc439 Mon Sep 17 00:00:00 2001 From: Juan Calderon-Perez <835733+gaby@users.noreply.github.com> Date: Thu, 18 Apr 2024 23:04:25 -0400 Subject: [PATCH 11/34] Update test-cloudflarekv.yml --- .github/workflows/test-cloudflarekv.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-cloudflarekv.yml b/.github/workflows/test-cloudflarekv.yml index d22d99fbb..89d00e874 100644 --- a/.github/workflows/test-cloudflarekv.yml +++ b/.github/workflows/test-cloudflarekv.yml @@ -6,10 +6,10 @@ on: - master - main paths: - - 'clickhouse/**' + - 'cloudflarekv/**' pull_request: paths: - - 'clickhouse/**' + - 'cloudflarekv/**' jobs: Tests: From f212e66726630a744771dd3c73b5369306329cef Mon Sep 17 00:00:00 2001 From: Lucas Lemos Date: Mon, 22 Apr 2024 11:04:32 -0300 Subject: [PATCH 12/34] chore: change clickhouse to port 9001 --- .github/workflows/benchmark.yml | 2 +- .github/workflows/test-clickhouse.yml | 2 +- clickhouse/clickhouse_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index cf3a51483..076f479dc 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -136,7 +136,7 @@ jobs: - name: Startup Clickhouse run: | - docker run -d -p 9000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server + docker run -d -p 9001:9000 --name clickhouse --ulimit nofile=262144:262144 clickhouse/clickhouse-server sleep 10 # Wait for Clickhouse to initialize - name: Setup Redis diff --git a/.github/workflows/test-clickhouse.yml b/.github/workflows/test-clickhouse.yml index 17e9aab7b..f9209d412 100644 --- a/.github/workflows/test-clickhouse.yml +++ b/.github/workflows/test-clickhouse.yml @@ -22,7 +22,7 @@ jobs: uses: actions/checkout@v4 - name: Startup Clickhouse run: | - docker run -d -p 9000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server + docker run -d -p 9001:9000 --name clickhouse --ulimit nofile=262144:262144 clickhouse/clickhouse-server sleep 30 - name: Install Go uses: actions/setup-go@v5 diff --git a/clickhouse/clickhouse_test.go b/clickhouse/clickhouse_test.go index 235c8b681..895a265f0 100644 --- a/clickhouse/clickhouse_test.go +++ b/clickhouse/clickhouse_test.go @@ -17,7 +17,7 @@ func getTestConnection(t TestOrBench) (*Storage, error) { client, err := New(Config{ Host: "127.0.0.1", - Port: 9000, + Port: 9001, Engine: "Memory", Table: "test_table", Clean: true, From bfd7ccc79e9a2b5c8d94508fa3d0f59124b66495 Mon Sep 17 00:00:00 2001 From: Lucas Lemos Date: Mon, 6 May 2024 14:42:39 -0300 Subject: [PATCH 13/34] refactor: change some namings and behaviors on Get --- clickhouse/clickhouse.go | 9 +++-- clickhouse/clickhouse_test.go | 64 +++++++++++++++++++++++++++++------ 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/clickhouse/clickhouse.go b/clickhouse/clickhouse.go index e43c79410..393cd3768 100644 --- a/clickhouse/clickhouse.go +++ b/clickhouse/clickhouse.go @@ -84,12 +84,17 @@ func (s *Storage) Get(key string) ([]byte, error) { })) err := s.session.Select(ctx, &resultSlice, selectDataString) if err != nil { - return nil, err + return []byte{}, err } + + if len(resultSlice) == 0 { + return []byte{}, nil + } + result := resultSlice[0] if !result.Expiration.IsZero() && result.Expiration.UTC().Unix() <= time.Now().UTC().Unix() { - return nil, nil + return []byte{}, nil } return []byte(result.Value), err diff --git a/clickhouse/clickhouse_test.go b/clickhouse/clickhouse_test.go index 895a265f0..8ae0452c6 100644 --- a/clickhouse/clickhouse_test.go +++ b/clickhouse/clickhouse_test.go @@ -17,7 +17,7 @@ func getTestConnection(t TestOrBench) (*Storage, error) { client, err := New(Config{ Host: "127.0.0.1", - Port: 9001, + Port: 9000, Engine: "Memory", Table: "test_table", Clean: true, @@ -26,13 +26,13 @@ func getTestConnection(t TestOrBench) (*Storage, error) { return client, err } -func TestConnection(t *testing.T) { +func Test_Connection(t *testing.T) { _, err := getTestConnection(t) require.NoError(t, err) } -func TestSet(t *testing.T) { +func Test_Set(t *testing.T) { client, err := getTestConnection(t) require.NoError(t, err) defer client.Close() @@ -41,7 +41,7 @@ func TestSet(t *testing.T) { require.NoError(t, err) } -func TestSetWithExp(t *testing.T) { +func Test_SetWithExp(t *testing.T) { client, err := getTestConnection(t) require.NoError(t, err) defer client.Close() @@ -50,7 +50,7 @@ func TestSetWithExp(t *testing.T) { require.NoError(t, err) } -func TestGet(t *testing.T) { +func Test_Get(t *testing.T) { client, err := getTestConnection(t) require.NoError(t, err) defer client.Close() @@ -65,7 +65,7 @@ func TestGet(t *testing.T) { assert.Equal(t, "somevalue", string(value)) } -func TestGetWithExp(t *testing.T) { +func Test_GetWithExp(t *testing.T) { client, err := getTestConnection(t) require.NoError(t, err) defer client.Close() @@ -84,10 +84,10 @@ func TestGetWithExp(t *testing.T) { value, err = client.Get("getsomekeywithexp") require.NoError(t, err) - assert.Nil(t, value) + assert.Equal(t, []byte{}, value) } -func TestDelete(t *testing.T) { +func Test_Delete(t *testing.T) { client, err := getTestConnection(t) require.NoError(t, err) defer client.Close() @@ -98,9 +98,14 @@ func TestDelete(t *testing.T) { err = client.Delete("somekeytodelete") require.NoError(t, err) + + value, err := client.Get("somekeytodelete") + + require.NoError(t, err) + assert.Equal(t, []byte{}, value) } -func TestReset(t *testing.T) { +func Test_Reset(t *testing.T) { client, err := getTestConnection(t) require.NoError(t, err) defer client.Close() @@ -111,9 +116,48 @@ func TestReset(t *testing.T) { err = client.Reset() require.NoError(t, err) + + value, err := client.Get("testkey") + + require.NoError(t, err) + assert.Equal(t, []byte{}, value) +} + +func Benchmark_Clickhouse_Set(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + client, err := getTestConnection(b) + require.NoError(b, err) + + defer client.Close() + + for i := 0; i < b.N; i++ { + err = client.Set("john", []byte("doe"), 0) + } + + require.NoError(b, err) +} + +func Benchmark_Clickhouse_Get(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + client, err := getTestConnection(b) + require.NoError(b, err) + + defer client.Close() + + err = client.Set("john", []byte("doe"), 0) + + for i := 0; i < b.N; i++ { + _, err = client.Get("john") + } + + require.NoError(b, err) } -func BenchmarkClickhouse(b *testing.B) { +func Benchmark_Clickhouse_Set_And_Delete(b *testing.B) { b.ReportAllocs() b.ResetTimer() From 0c53f4588b0ae04ea2b447092400960afeafd397 Mon Sep 17 00:00:00 2001 From: Lucas Lemos Date: Mon, 6 May 2024 14:51:10 -0300 Subject: [PATCH 14/34] chore: return nil for error on storage creation Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- clickhouse/clickhouse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse/clickhouse.go b/clickhouse/clickhouse.go index 393cd3768..98ffd1861 100644 --- a/clickhouse/clickhouse.go +++ b/clickhouse/clickhouse.go @@ -23,7 +23,7 @@ func New(configuration Config) (*Storage, error) { conn, err := driver.Open(&cfg) if err != nil { - return &Storage{}, err + return nil, err } ctx := driver.Context(context.Background(), driver.WithParameters(driver.Parameters{ From ca256c013f258be6279c0df7906fd047c9ac6c3e Mon Sep 17 00:00:00 2001 From: Lucas Lemos Date: Mon, 6 May 2024 14:51:31 -0300 Subject: [PATCH 15/34] chore: return nil for error on storage creation Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- clickhouse/clickhouse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse/clickhouse.go b/clickhouse/clickhouse.go index 98ffd1861..9792e70f6 100644 --- a/clickhouse/clickhouse.go +++ b/clickhouse/clickhouse.go @@ -18,7 +18,7 @@ type Storage struct { func New(configuration Config) (*Storage, error) { cfg, engine, err := defaultConfig(configuration) if err != nil { - return &Storage{}, err + return nil, err } conn, err := driver.Open(&cfg) From 11d7b5f8864c83db606c489adaa7edcd9927d558 Mon Sep 17 00:00:00 2001 From: Lucas Lemos Date: Mon, 6 May 2024 14:52:53 -0300 Subject: [PATCH 16/34] refactor: improve comparision beteween times Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- clickhouse/clickhouse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse/clickhouse.go b/clickhouse/clickhouse.go index 9792e70f6..077b96e61 100644 --- a/clickhouse/clickhouse.go +++ b/clickhouse/clickhouse.go @@ -93,7 +93,7 @@ func (s *Storage) Get(key string) ([]byte, error) { result := resultSlice[0] - if !result.Expiration.IsZero() && result.Expiration.UTC().Unix() <= time.Now().UTC().Unix() { + if !result.Expiration.IsZero() && result.Expiration.Before(time.Now().UTC()) { return []byte{}, nil } From bdf87e37188370826072378991de5726f54935b0 Mon Sep 17 00:00:00 2001 From: Lucas Lemos Date: Mon, 6 May 2024 15:30:50 -0300 Subject: [PATCH 17/34] refactor: check for connection errors before returning storage --- clickhouse/clickhouse.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/clickhouse/clickhouse.go b/clickhouse/clickhouse.go index 077b96e61..4bd60eb2e 100644 --- a/clickhouse/clickhouse.go +++ b/clickhouse/clickhouse.go @@ -41,11 +41,16 @@ func New(configuration Config) (*Storage, error) { } } + err = conn.Ping(ctx) + if err != nil { + return nil, err + } + return &Storage{ session: conn, context: ctx, table: configuration.Table, - }, conn.Ping(ctx) + }, nil } func (s *Storage) Set(key string, value []byte, expiration time.Duration) error { From 3ef00028e6c97cead7045e2fc96bccf6e108b8ed Mon Sep 17 00:00:00 2001 From: luk3skyw4lker Date: Thu, 20 Jun 2024 14:28:00 -0300 Subject: [PATCH 18/34] feat: add zstd compression to columns --- clickhouse/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clickhouse/config.go b/clickhouse/config.go index f4b27dca0..6ee9ed64f 100644 --- a/clickhouse/config.go +++ b/clickhouse/config.go @@ -106,8 +106,8 @@ const insertDataString = ` const createTableString = ` CREATE TABLE IF NOT EXISTS {table:Identifier} ( - key String - , value String - , expiration Datetime + key String CODEC(ZSTD(1)) + , value String CODEC(ZSTD(1)) + , expiration Datetime CODEC(ZSTD(1)) ) ENGINE=%s ` From c62c0af76c754b7facf95ddc4c989d4db4d1352a Mon Sep 17 00:00:00 2001 From: luk3skyw4lker Date: Thu, 20 Jun 2024 14:28:11 -0300 Subject: [PATCH 19/34] fix: broken tests and benchmarks --- clickhouse/clickhouse_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse/clickhouse_test.go b/clickhouse/clickhouse_test.go index 8ae0452c6..a98acad07 100644 --- a/clickhouse/clickhouse_test.go +++ b/clickhouse/clickhouse_test.go @@ -17,7 +17,7 @@ func getTestConnection(t TestOrBench) (*Storage, error) { client, err := New(Config{ Host: "127.0.0.1", - Port: 9000, + Port: 9001, Engine: "Memory", Table: "test_table", Clean: true, From c70e44944d669f8d28b951ff9dabfc9e2f675d0d Mon Sep 17 00:00:00 2001 From: luk3skyw4lker Date: Thu, 20 Jun 2024 18:47:16 -0300 Subject: [PATCH 20/34] refactor: minor code rabbit changes --- clickhouse/clickhouse_test.go | 90 ++++++++++++++++++++++++++++------- clickhouse/config.go | 16 ++++--- 2 files changed, 81 insertions(+), 25 deletions(-) diff --git a/clickhouse/clickhouse_test.go b/clickhouse/clickhouse_test.go index a98acad07..f56ddd5ee 100644 --- a/clickhouse/clickhouse_test.go +++ b/clickhouse/clickhouse_test.go @@ -12,28 +12,34 @@ type TestOrBench interface { Helper() } -func getTestConnection(t TestOrBench) (*Storage, error) { +func getTestConnection(t TestOrBench, cfg Config) (*Storage, error) { t.Helper() - client, err := New(Config{ - Host: "127.0.0.1", - Port: 9001, - Engine: "Memory", - Table: "test_table", - Clean: true, - }) + client, err := New(cfg) return client, err } func Test_Connection(t *testing.T) { - _, err := getTestConnection(t) + _, err := getTestConnection(t, Config{ + Host: "127.0.0.1", + Port: 9001, + Engine: Memory, + Table: "test_table", + Clean: true, + }) require.NoError(t, err) } func Test_Set(t *testing.T) { - client, err := getTestConnection(t) + client, err := getTestConnection(t, Config{ + Host: "127.0.0.1", + Port: 9001, + Engine: Memory, + Table: "test_table", + Clean: true, + }) require.NoError(t, err) defer client.Close() @@ -42,7 +48,13 @@ func Test_Set(t *testing.T) { } func Test_SetWithExp(t *testing.T) { - client, err := getTestConnection(t) + client, err := getTestConnection(t, Config{ + Host: "127.0.0.1", + Port: 9001, + Engine: Memory, + Table: "test_table", + Clean: true, + }) require.NoError(t, err) defer client.Close() @@ -51,7 +63,13 @@ func Test_SetWithExp(t *testing.T) { } func Test_Get(t *testing.T) { - client, err := getTestConnection(t) + client, err := getTestConnection(t, Config{ + Host: "127.0.0.1", + Port: 9001, + Engine: Memory, + Table: "test_table", + Clean: true, + }) require.NoError(t, err) defer client.Close() @@ -66,7 +84,13 @@ func Test_Get(t *testing.T) { } func Test_GetWithExp(t *testing.T) { - client, err := getTestConnection(t) + client, err := getTestConnection(t, Config{ + Host: "127.0.0.1", + Port: 9001, + Engine: Memory, + Table: "test_table", + Clean: true, + }) require.NoError(t, err) defer client.Close() @@ -88,7 +112,13 @@ func Test_GetWithExp(t *testing.T) { } func Test_Delete(t *testing.T) { - client, err := getTestConnection(t) + client, err := getTestConnection(t, Config{ + Host: "127.0.0.1", + Port: 9001, + Engine: Memory, + Table: "test_table", + Clean: true, + }) require.NoError(t, err) defer client.Close() @@ -106,7 +136,13 @@ func Test_Delete(t *testing.T) { } func Test_Reset(t *testing.T) { - client, err := getTestConnection(t) + client, err := getTestConnection(t, Config{ + Host: "127.0.0.1", + Port: 9001, + Engine: Memory, + Table: "test_table", + Clean: true, + }) require.NoError(t, err) defer client.Close() @@ -127,7 +163,13 @@ func Benchmark_Clickhouse_Set(b *testing.B) { b.ReportAllocs() b.ResetTimer() - client, err := getTestConnection(b) + client, err := getTestConnection(b, Config{ + Host: "127.0.0.1", + Port: 9001, + Engine: Memory, + Table: "test_table", + Clean: true, + }) require.NoError(b, err) defer client.Close() @@ -143,7 +185,13 @@ func Benchmark_Clickhouse_Get(b *testing.B) { b.ReportAllocs() b.ResetTimer() - client, err := getTestConnection(b) + client, err := getTestConnection(b, Config{ + Host: "127.0.0.1", + Port: 9001, + Engine: Memory, + Table: "test_table", + Clean: true, + }) require.NoError(b, err) defer client.Close() @@ -161,7 +209,13 @@ func Benchmark_Clickhouse_Set_And_Delete(b *testing.B) { b.ReportAllocs() b.ResetTimer() - client, err := getTestConnection(b) + client, err := getTestConnection(b, Config{ + Host: "127.0.0.1", + Port: 9001, + Engine: Memory, + Table: "test_table", + Clean: true, + }) require.NoError(b, err) defer client.Close() diff --git a/clickhouse/config.go b/clickhouse/config.go index 6ee9ed64f..9b3aa28a2 100644 --- a/clickhouse/config.go +++ b/clickhouse/config.go @@ -9,17 +9,19 @@ import ( driver "github.com/ClickHouse/clickhouse-go/v2" ) +type ClickhouseEngine string + type schema struct { Value string `ch:"value"` Expiration time.Time `ch:"expiration"` } const ( - Memory = "Memory" - MergeTree = "MergeTree" - StripeLog = "StripeLog" - TinyLog = "TinyLog" - Log = "Log" + Memory ClickhouseEngine = "Memory" + MergeTree ClickhouseEngine = "MergeTree" + StripeLog ClickhouseEngine = "StripeLog" + TinyLog ClickhouseEngine = "TinyLog" + Log ClickhouseEngine = "Log" ) // Config defines configuration options for Clickhouse connection. @@ -37,7 +39,7 @@ type Config struct { // The name of the table that will store the data Table string // The engine that should be used in the table - Engine string + Engine ClickhouseEngine // Should start a clean table, default false Clean bool // TLS configuration, default nil @@ -48,7 +50,7 @@ type Config struct { Debugf func(format string, v ...any) } -func defaultConfig(configuration Config) (driver.Options, string, error) { +func defaultConfig(configuration Config) (driver.Options, ClickhouseEngine, error) { if configuration.Table == "" { return driver.Options{}, "", errors.New("table name not provided") } From 0887a66d653bcec06a46a5232eb1014f874c22f0 Mon Sep 17 00:00:00 2001 From: luk3skyw4lker Date: Tue, 25 Jun 2024 09:12:28 -0300 Subject: [PATCH 21/34] chore: update clickhouse driver --- clickhouse/go.mod | 14 +++++++------- clickhouse/go.sum | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/clickhouse/go.mod b/clickhouse/go.mod index 4b5909378..ef8ee63fd 100644 --- a/clickhouse/go.mod +++ b/clickhouse/go.mod @@ -3,12 +3,12 @@ module github.com/gofiber/storage/clickhouse go 1.21 require ( - github.com/ClickHouse/clickhouse-go/v2 v2.20.0 - github.com/stretchr/testify v1.8.4 + github.com/ClickHouse/clickhouse-go/v2 v2.26.0 + github.com/stretchr/testify v1.9.0 ) require ( - github.com/ClickHouse/ch-go v0.61.3 // indirect + github.com/ClickHouse/ch-go v0.61.5 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-faster/city v1.0.1 // indirect @@ -20,9 +20,9 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/segmentio/asm v1.2.0 // indirect - github.com/shopspring/decimal v1.3.1 // indirect - go.opentelemetry.io/otel v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect - golang.org/x/sys v0.17.0 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + go.opentelemetry.io/otel v1.26.0 // indirect + go.opentelemetry.io/otel/trace v1.26.0 // indirect + golang.org/x/sys v0.18.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/clickhouse/go.sum b/clickhouse/go.sum index c8427c637..1928cca87 100644 --- a/clickhouse/go.sum +++ b/clickhouse/go.sum @@ -1,7 +1,12 @@ github.com/ClickHouse/ch-go v0.61.3 h1:MmBwUhXrAOBZK7n/sWBzq6FdIQ01cuF2SaaO8KlDRzI= github.com/ClickHouse/ch-go v0.61.3/go.mod h1:1PqXjMz/7S1ZUaKvwPA3i35W2bz2mAMFeCi6DIXgGwQ= +github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= +github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= +github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= github.com/ClickHouse/clickhouse-go/v2 v2.20.0 h1:bvlLQ31XJfl7MxIqAq2l1G6JhHYzqEXdvfpMeU6bkKc= github.com/ClickHouse/clickhouse-go/v2 v2.20.0/go.mod h1:VQfyA+tCwCRw2G7ogfY8V0fq/r0yJWzy8UDrjiP/Lbs= +github.com/ClickHouse/clickhouse-go/v2 v2.26.0 h1:j4/y6NYaCcFkJwN/TU700ebW+nmsIy34RmUAAcZKy9w= +github.com/ClickHouse/clickhouse-go/v2 v2.26.0/go.mod h1:iDTViXk2Fgvf1jn2dbJd1ys+fBkdD1UMRnXlwmhijhQ= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -47,10 +52,13 @@ github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= @@ -61,8 +69,12 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -86,6 +98,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 1f3a53a7556722ddf6472823866d39277f67bd39 Mon Sep 17 00:00:00 2001 From: luk3skyw4lker Date: Tue, 25 Jun 2024 09:13:11 -0300 Subject: [PATCH 22/34] chore: update supported versions of Go in README --- clickhouse/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse/README.md b/clickhouse/README.md index e9893f081..a630e15c0 100644 --- a/clickhouse/README.md +++ b/clickhouse/README.md @@ -20,7 +20,7 @@ func (s *Storage) Conn() *Session ``` ### Installation -Clickhouse is supported on Go versions 1.19 and above: +Clickhouse is supported on the latest two versions of Go: Install the clickhouse implementation: ```bash From bf0d8fc9d00f2b827c775e5de61fc95ed87d92c2 Mon Sep 17 00:00:00 2001 From: luk3skyw4lker Date: Tue, 25 Jun 2024 09:17:57 -0300 Subject: [PATCH 23/34] chore: add release-drafter-clickhouse.yml and config --- .github/release-drafter-clickhouse.yml | 50 +++++++++++++++++++ .../workflows/release-drafter-clickhouse.yml | 19 +++++++ 2 files changed, 69 insertions(+) create mode 100644 .github/release-drafter-clickhouse.yml create mode 100644 .github/workflows/release-drafter-clickhouse.yml diff --git a/.github/release-drafter-clickhouse.yml b/.github/release-drafter-clickhouse.yml new file mode 100644 index 000000000..c08b7f252 --- /dev/null +++ b/.github/release-drafter-clickhouse.yml @@ -0,0 +1,50 @@ +name-template: 'ClickHouse - v$RESOLVED_VERSION' +tag-template: 'clickhouse/v$RESOLVED_VERSION' +tag-prefix: clickhouse/v +include-paths: + - clickhouse +categories: + - title: 'โ— Breaking Changes' + labels: + - 'โ— BreakingChange' + - title: '๐Ÿš€ New' + labels: + - 'โœ๏ธ Feature' + - title: '๐Ÿงน Updates' + labels: + - '๐Ÿงน Updates' + - '๐Ÿค– Dependencies' + - title: '๐Ÿ› Fixes' + labels: + - 'โ˜ข๏ธ Bug' + - title: '๐Ÿ“š Documentation' + labels: + - '๐Ÿ“’ Documentation' +change-template: '- $TITLE (#$NUMBER)' +change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks. +exclude-contributors: + - dependabot + - dependabot[bot] +version-resolver: + major: + labels: + - 'major' + - 'โ— BreakingChange' + minor: + labels: + - 'minor' + - 'โœ๏ธ Feature' + patch: + labels: + - 'patch' + - '๐Ÿ“’ Documentation' + - 'โ˜ข๏ธ Bug' + - '๐Ÿค– Dependencies' + - '๐Ÿงน Updates' + default: patch +template: | + $CHANGES + + **Full Changelog**: https://github.com/$OWNER/$REPOSITORY/compare/$PREVIOUS_TAG...clickhouse/v$RESOLVED_VERSION + + Thank you $CONTRIBUTORS for making this update possible. diff --git a/.github/workflows/release-drafter-clickhouse.yml b/.github/workflows/release-drafter-clickhouse.yml new file mode 100644 index 000000000..62043170e --- /dev/null +++ b/.github/workflows/release-drafter-clickhouse.yml @@ -0,0 +1,19 @@ +name: Release Drafter Clickhouse +on: + push: + # branches to consider in the event; optional, defaults to all + branches: + - master + - main + paths: + - 'clickhouse/**' +jobs: + draft_release_clickhouse: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: release-drafter/release-drafter@v6 + with: + config-name: release-drafter-clickhouse.yml + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From d27ee3679c434bf148f6b5500e4d682ccf7a9605 Mon Sep 17 00:00:00 2001 From: luk3skyw4lker Date: Tue, 25 Jun 2024 09:18:07 -0300 Subject: [PATCH 24/34] chore: add clickhouse to dependabot.yml --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 7e154f353..5c3f0928d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -164,3 +164,9 @@ updates: - "๐Ÿค– Dependencies" schedule: interval: "daily" + - package-ecosystem: "gomod" + directory: "/clickhouse/" # Location of package manifests + labels: + - "๐Ÿค– Dependencies" + schedule: + interval: "daily" From 292cee65393ded5d5d499df48946a2b17ee17641 Mon Sep 17 00:00:00 2001 From: luk3skyw4lker Date: Tue, 25 Jun 2024 09:18:22 -0300 Subject: [PATCH 25/34] docs: add clickhouse README reference docs --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 532bebabf..40f61d30b 100644 --- a/README.md +++ b/README.md @@ -74,3 +74,4 @@ type Storage interface { - [S3](./s3/README.md) - [ScyllaDB](./scylladb/README.md) - [SQLite3](./sqlite3/README.md) +- [ClickHouse](./clickhouse/README.md) From 4311191155ec1a21207e7d7fbad5d82266eb05c2 Mon Sep 17 00:00:00 2001 From: Lucas Lemos Date: Wed, 26 Jun 2024 08:43:37 -0300 Subject: [PATCH 26/34] docs: update clickhouse/README.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- clickhouse/README.md | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/clickhouse/README.md b/clickhouse/README.md index a630e15c0..d6f89db36 100644 --- a/clickhouse/README.md +++ b/clickhouse/README.md @@ -78,28 +78,28 @@ store, err := clickhouse.New(coherence.Config{ ```go // Config defines configuration options for Clickhouse connection. type Config struct { - // The host of the database. Ex: 127.0.0.1 - Host string - // The port where the database is supposed to listen to. Ex: 9000 - Port int - // The database that the connection should authenticate from - Database string - // The username to be used in the authentication - Username string - // The password to be used in the authentication - Password string - // The name of the table that will store the data - Table string - // The engine that should be used in the table - Engine string - // Should start a clean table, default false - Clean bool - // TLS configuration, default nil - TLSConfig *tls.Config - // Should the connection be in debug mode, default false - Debug bool - // The function to use with the debug config, default print function. It only works when debug is true - Debugf func(format string, v ...any) + // The host of the database. Ex: 127.0.0.1 + Host string + // The port where the database is supposed to listen to. Ex: 9000 + Port int + // The database that the connection should authenticate from + Database string + // The username to be used in the authentication + Username string + // The password to be used in the authentication + Password string + // The name of the table that will store the data + Table string + // The engine that should be used in the table + Engine string + // Should start a clean table, default false + Clean bool + // TLS configuration, default nil + TLSConfig *tls.Config + // Should the connection be in debug mode, default false + Debug bool + // The function to use with the debug config, default print function. It only works when debug is true + Debugf func(format string, v ...any) } ``` From f4ee2db62cc6eb6093f2d5872cd956b597699d67 Mon Sep 17 00:00:00 2001 From: Lucas Lemos Date: Wed, 26 Jun 2024 08:56:14 -0300 Subject: [PATCH 27/34] docs: update markdown formatting --- clickhouse/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/clickhouse/README.md b/clickhouse/README.md index d6f89db36..1a14d959f 100644 --- a/clickhouse/README.md +++ b/clickhouse/README.md @@ -1,7 +1,9 @@ # Clickhouse + A Clickhouse storage driver using [https://github.com/ClickHouse/clickhouse-go](https://github.com/ClickHouse/clickhouse-go). ### Table of Contents + - [Signatures](#signatures) - [Installation](#installation) - [Examples](#examples) @@ -9,6 +11,7 @@ A Clickhouse storage driver using [https://github.com/ClickHouse/clickhouse-go]( - [Default Config](#default-config) ### Signatures + ```go func New(config ...Config) (*Storage, error) func (s *Storage) Get(key string) ([]byte, error) @@ -20,6 +23,7 @@ func (s *Storage) Conn() *Session ``` ### Installation + Clickhouse is supported on the latest two versions of Go: Install the clickhouse implementation: @@ -104,6 +108,7 @@ type Config struct { ``` ### Default Config + ```go var DefaultConfig = Config{ Host: "localhost", From d872a8574b17a623d96667824301e7b414fcac52 Mon Sep 17 00:00:00 2001 From: luk3skyw4lker Date: Wed, 26 Jun 2024 11:24:50 -0300 Subject: [PATCH 28/34] chore: tidying up modules --- clickhouse/go.sum | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/clickhouse/go.sum b/clickhouse/go.sum index 1928cca87..41017a588 100644 --- a/clickhouse/go.sum +++ b/clickhouse/go.sum @@ -1,10 +1,5 @@ -github.com/ClickHouse/ch-go v0.61.3 h1:MmBwUhXrAOBZK7n/sWBzq6FdIQ01cuF2SaaO8KlDRzI= -github.com/ClickHouse/ch-go v0.61.3/go.mod h1:1PqXjMz/7S1ZUaKvwPA3i35W2bz2mAMFeCi6DIXgGwQ= github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= -github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= -github.com/ClickHouse/clickhouse-go/v2 v2.20.0 h1:bvlLQ31XJfl7MxIqAq2l1G6JhHYzqEXdvfpMeU6bkKc= -github.com/ClickHouse/clickhouse-go/v2 v2.20.0/go.mod h1:VQfyA+tCwCRw2G7ogfY8V0fq/r0yJWzy8UDrjiP/Lbs= github.com/ClickHouse/clickhouse-go/v2 v2.26.0 h1:j4/y6NYaCcFkJwN/TU700ebW+nmsIy34RmUAAcZKy9w= github.com/ClickHouse/clickhouse-go/v2 v2.26.0/go.mod h1:iDTViXk2Fgvf1jn2dbJd1ys+fBkdD1UMRnXlwmhijhQ= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= @@ -50,14 +45,11 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= -github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -67,12 +59,8 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -96,8 +84,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= From f27f7b10558eab128a9af769e554933b0eaf6a5d Mon Sep 17 00:00:00 2001 From: luk3skyw4lker Date: Mon, 1 Jul 2024 08:58:54 -0300 Subject: [PATCH 29/34] fix: time.Time.IsZero returning false negative --- clickhouse/clickhouse.go | 84 ++++++++++++++++++----------------- clickhouse/clickhouse_test.go | 8 ++-- clickhouse/config.go | 5 ++- clickhouse/go.sum | 2 + 4 files changed, 53 insertions(+), 46 deletions(-) diff --git a/clickhouse/clickhouse.go b/clickhouse/clickhouse.go index 4bd60eb2e..5f982f3f9 100644 --- a/clickhouse/clickhouse.go +++ b/clickhouse/clickhouse.go @@ -2,6 +2,8 @@ package clickhouse import ( "context" + "database/sql" + "errors" "fmt" "time" @@ -26,17 +28,15 @@ func New(configuration Config) (*Storage, error) { return nil, err } - ctx := driver.Context(context.Background(), driver.WithParameters(driver.Parameters{ - "table": configuration.Table, - })) + ctx := context.Background() - if configuration.Clean { - queryWithEngine := fmt.Sprintf(createTableString, engine) - if err := conn.Exec(ctx, queryWithEngine); err != nil { - return &Storage{}, err - } + queryWithEngine := fmt.Sprintf(createTableString, engine) + if err := conn.Exec(ctx, queryWithEngine, driver.Named("table", configuration.Table)); err != nil { + return &Storage{}, err + } - if err := conn.Exec(ctx, resetDataString); err != nil { + if configuration.Clean { + if err := conn.Exec(ctx, resetDataString, driver.Named("table", configuration.Table)); err != nil { return &Storage{}, err } } @@ -63,59 +63,63 @@ func (s *Storage) Set(key string, value []byte, expiration time.Duration) error exp = time.Now().Add(expiration).UTC() } - ctx := driver.Context(s.context, driver.WithParameters(driver.Parameters{ - "key": key, - "value": string(value), - "expiration": exp.Format("2006-01-02 15:04:05"), - "table": s.table, - })) - - err := s. + return s. session. Exec( - ctx, + s.context, insertDataString, + driver.Named("table", s.table), + driver.Named("key", key), + driver.Named("value", string(value)), + driver.Named("expiration", exp.Format("2006-01-02 15:04:05")), ) - - return err } func (s *Storage) Get(key string) ([]byte, error) { - var resultSlice []schema - - ctx := driver.Context(s.context, driver.WithParameters(driver.Parameters{ - "key": key, - "table": s.table, - })) - err := s.session.Select(ctx, &resultSlice, selectDataString) - if err != nil { - return []byte{}, err + if len(key) == 0 { + return []byte{}, nil } - if len(resultSlice) == 0 { - return []byte{}, nil + var result schema + + row := s.session.QueryRow( + s.context, + selectDataString, + driver.Named("table", s.table), + driver.Named("key", key), + ) + if row.Err() != nil { + return []byte{}, row.Err() } - result := resultSlice[0] + if err := row.ScanStruct(&result); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return []byte{}, nil + } - if !result.Expiration.IsZero() && result.Expiration.Before(time.Now().UTC()) { + return []byte{}, err + } + + // The result.Expiration.IsZero() was returning a false value even when the time was + // set to be the zero value of the time.Time struct (Jan 1st 1970, 00:00:00 UTC) + // so we had to change the comparision + if !time.Unix(0, 0).Equal(result.Expiration) && result.Expiration.Before(time.Now().UTC()) { return []byte{}, nil } - return []byte(result.Value), err + return []byte(result.Value), nil } func (s *Storage) Delete(key string) error { - ctx := driver.Context(s.context, driver.WithParameters(driver.Parameters{ - "table": s.table, - "key": key, - })) + if len(key) == 0 { + return nil + } - return s.session.Exec(ctx, deleteDataString) + return s.session.Exec(s.context, deleteDataString, driver.Named("table", s.table), driver.Named("key", key)) } func (s *Storage) Reset() error { - return s.session.Exec(s.context, resetDataString) + return s.session.Exec(s.context, resetDataString, driver.Named("table", s.table)) } func (s *Storage) Close() error { diff --git a/clickhouse/clickhouse_test.go b/clickhouse/clickhouse_test.go index f56ddd5ee..f1278ce2d 100644 --- a/clickhouse/clickhouse_test.go +++ b/clickhouse/clickhouse_test.go @@ -47,7 +47,7 @@ func Test_Set(t *testing.T) { require.NoError(t, err) } -func Test_SetWithExp(t *testing.T) { +func Test_Set_With_Exp(t *testing.T) { client, err := getTestConnection(t, Config{ Host: "127.0.0.1", Port: 9001, @@ -83,7 +83,7 @@ func Test_Get(t *testing.T) { assert.Equal(t, "somevalue", string(value)) } -func Test_GetWithExp(t *testing.T) { +func Test_Get_With_Exp(t *testing.T) { client, err := getTestConnection(t, Config{ Host: "127.0.0.1", Port: 9001, @@ -94,7 +94,7 @@ func Test_GetWithExp(t *testing.T) { require.NoError(t, err) defer client.Close() - err = client.Set("getsomekeywithexp", []byte("somevalue"), time.Second*5) + err = client.Set("getsomekeywithexp", []byte("somevalue"), time.Second*2) require.NoError(t, err) value, err := client.Get("getsomekeywithexp") @@ -103,7 +103,7 @@ func Test_GetWithExp(t *testing.T) { assert.NotNil(t, value) assert.Equal(t, "somevalue", string(value)) - time.Sleep(time.Second * 10) + time.Sleep(time.Second * 5) value, err = client.Get("getsomekeywithexp") diff --git a/clickhouse/config.go b/clickhouse/config.go index 9b3aa28a2..d9b820ce3 100644 --- a/clickhouse/config.go +++ b/clickhouse/config.go @@ -4,6 +4,7 @@ import ( "crypto/tls" "errors" "fmt" + "log" "time" driver "github.com/ClickHouse/clickhouse-go/v2" @@ -64,7 +65,7 @@ func defaultConfig(configuration Config) (driver.Options, ClickhouseEngine, erro } if configuration.Engine == "" { - configuration.Engine = "Memory" + configuration.Engine = Memory } config := driver.Options{ @@ -84,7 +85,7 @@ func defaultConfig(configuration Config) (driver.Options, ClickhouseEngine, erro } if configuration.Debug && config.Debugf == nil { - config.Debugf = func(format string, v ...any) { fmt.Printf(format, v...) } + config.Debugf = log.Printf } return config, configuration.Engine, nil diff --git a/clickhouse/go.sum b/clickhouse/go.sum index 41017a588..9a025ec30 100644 --- a/clickhouse/go.sum +++ b/clickhouse/go.sum @@ -1,5 +1,7 @@ github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= +github.com/ClickHouse/clickhouse-go/v2 v2.24.0 h1:L/n/pVVpk95KtkHOiKuSnO7cu2ckeW4gICbbOh5qs74= +github.com/ClickHouse/clickhouse-go/v2 v2.24.0/go.mod h1:iDTViXk2Fgvf1jn2dbJd1ys+fBkdD1UMRnXlwmhijhQ= github.com/ClickHouse/clickhouse-go/v2 v2.26.0 h1:j4/y6NYaCcFkJwN/TU700ebW+nmsIy34RmUAAcZKy9w= github.com/ClickHouse/clickhouse-go/v2 v2.26.0/go.mod h1:iDTViXk2Fgvf1jn2dbJd1ys+fBkdD1UMRnXlwmhijhQ= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= From e744bc09fae141e2f753176149eb0c442b14d931 Mon Sep 17 00:00:00 2001 From: luk3skyw4lker Date: Mon, 1 Jul 2024 12:52:50 -0300 Subject: [PATCH 30/34] refactor: change some error returns to nil instead of &Storage --- clickhouse/clickhouse.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/clickhouse/clickhouse.go b/clickhouse/clickhouse.go index 5f982f3f9..d033afede 100644 --- a/clickhouse/clickhouse.go +++ b/clickhouse/clickhouse.go @@ -32,17 +32,16 @@ func New(configuration Config) (*Storage, error) { queryWithEngine := fmt.Sprintf(createTableString, engine) if err := conn.Exec(ctx, queryWithEngine, driver.Named("table", configuration.Table)); err != nil { - return &Storage{}, err + return nil, err } if configuration.Clean { if err := conn.Exec(ctx, resetDataString, driver.Named("table", configuration.Table)); err != nil { - return &Storage{}, err + return nil, err } } - err = conn.Ping(ctx) - if err != nil { + if err := conn.Ping(ctx); err != nil { return nil, err } From f9d76aa3d6b1761d31acf57e80b547d7afd33da4 Mon Sep 17 00:00:00 2001 From: Juan Calderon-Perez <835733+gaby@users.noreply.github.com> Date: Wed, 3 Jul 2024 08:18:55 -0400 Subject: [PATCH 31/34] Update clickhouse README.md --- clickhouse/README.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/clickhouse/README.md b/clickhouse/README.md index 1a14d959f..772f5527f 100644 --- a/clickhouse/README.md +++ b/clickhouse/README.md @@ -44,13 +44,11 @@ docker run -d -p 9000:9000 --name some-clickhouse-server --ulimit nofile=262144: After running this command you're ready to start using the storage and connecting to the database. ### Examples -Import the storage package. -```go -import "github.com/gofiber/storage/coherence" -``` -You can use the following possibilities to create a storage: +You can use the following options to create a clickhouse storage driver: ```go +import "github.com/gofiber/storage/clickhouse" + // Initialize default config, to connect to localhost:9000 using the memory engine and with a clean table. store, err := clickhouse.New(clickhouse.Config{ Host: "localhost", @@ -69,7 +67,7 @@ store, err := clickhouse.New(clickhouse.Config{ // Initialize to connect with TLS enabled with your own tls.Config and with clean table. tlsConfig := config := &tls.Config{...} -store, err := clickhouse.New(coherence.Config{ +store, err := clickhouse.New(clickhouse.Config{ Host: "some-ip-address", Port: 9000, Clean: true, @@ -116,4 +114,4 @@ var DefaultConfig = Config{ Engine: "Memory", Clean: false, } -``` \ No newline at end of file +``` From 337546b0774eed0abb0032a5cda664a0e57a3d2b Mon Sep 17 00:00:00 2001 From: Juan Calderon-Perez <835733+gaby@users.noreply.github.com> Date: Wed, 3 Jul 2024 08:20:38 -0400 Subject: [PATCH 32/34] Update comments in clickhouse.go --- clickhouse/clickhouse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clickhouse/clickhouse.go b/clickhouse/clickhouse.go index d033afede..1056bc7b9 100644 --- a/clickhouse/clickhouse.go +++ b/clickhouse/clickhouse.go @@ -101,7 +101,7 @@ func (s *Storage) Get(key string) ([]byte, error) { // The result.Expiration.IsZero() was returning a false value even when the time was // set to be the zero value of the time.Time struct (Jan 1st 1970, 00:00:00 UTC) - // so we had to change the comparision + // so we had to change the comparison if !time.Unix(0, 0).Equal(result.Expiration) && result.Expiration.Before(time.Now().UTC()) { return []byte{}, nil } From ea065c4ab86190a7db224f3979edaf60216bdcd0 Mon Sep 17 00:00:00 2001 From: Juan Calderon-Perez <835733+gaby@users.noreply.github.com> Date: Wed, 3 Jul 2024 08:22:02 -0400 Subject: [PATCH 33/34] Enable -race for clickhouse tests --- .github/workflows/test-clickhouse.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-clickhouse.yml b/.github/workflows/test-clickhouse.yml index f9209d412..a56a54de8 100644 --- a/.github/workflows/test-clickhouse.yml +++ b/.github/workflows/test-clickhouse.yml @@ -29,4 +29,4 @@ jobs: with: go-version: '${{ matrix.go-version }}' - name: Run Test - run: cd ./clickhouse && go clean -testcache && go test ./... -v + run: cd ./clickhouse && go clean -testcache && go test ./... -v -race From 39412ce785f86c6c9bc95e37891e9d39c9232be9 Mon Sep 17 00:00:00 2001 From: Juan Calderon-Perez Date: Wed, 3 Jul 2024 08:33:57 -0400 Subject: [PATCH 34/34] Update go.mod for Clickhouse --- clickhouse/go.mod | 8 ++++---- clickhouse/go.sum | 18 ++++++++---------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/clickhouse/go.mod b/clickhouse/go.mod index ef8ee63fd..f40124aed 100644 --- a/clickhouse/go.mod +++ b/clickhouse/go.mod @@ -14,15 +14,15 @@ require ( github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/klauspost/compress v1.17.7 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/paulmach/orb v0.11.1 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect - go.opentelemetry.io/otel v1.26.0 // indirect - go.opentelemetry.io/otel/trace v1.26.0 // indirect - golang.org/x/sys v0.18.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect + golang.org/x/sys v0.21.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/clickhouse/go.sum b/clickhouse/go.sum index 9a025ec30..46d905986 100644 --- a/clickhouse/go.sum +++ b/clickhouse/go.sum @@ -1,7 +1,5 @@ github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= -github.com/ClickHouse/clickhouse-go/v2 v2.24.0 h1:L/n/pVVpk95KtkHOiKuSnO7cu2ckeW4gICbbOh5qs74= -github.com/ClickHouse/clickhouse-go/v2 v2.24.0/go.mod h1:iDTViXk2Fgvf1jn2dbJd1ys+fBkdD1UMRnXlwmhijhQ= github.com/ClickHouse/clickhouse-go/v2 v2.26.0 h1:j4/y6NYaCcFkJwN/TU700ebW+nmsIy34RmUAAcZKy9w= github.com/ClickHouse/clickhouse-go/v2 v2.26.0/go.mod h1:iDTViXk2Fgvf1jn2dbJd1ys+fBkdD1UMRnXlwmhijhQ= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= @@ -25,8 +23,8 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= -github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -61,10 +59,10 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= -go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= -go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= -go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= -go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -86,8 +84,8 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=