Update dependencies, add date truncation e2e tests

.github/workflows/tests.yml

@@ -29,7 +29,7 @@ jobs:
       fail-fast: true
       matrix:
         go:
-          - "1.20"
+          - "1.22"
         clickhouse:
           - "latest"
     steps:

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 * Git
 * Make (see [Makefile](./Makefile) for all available commands)
-* Go 1.20+
+* Go 1.22+
 
 ## Install Protoc and Go plugin
 

destination/db/values/values.go

@@ -102,12 +102,19 @@ func Parse(colName string, colType pb.DataType, val string) (any, error) {
 		}
 		// Supported range of values: [1900-01-01 00:00:00, 2299-12-31 23:59:59.99999999].
 		// See https://clickhouse.com/docs/en/sql-reference/data-types/datetime64
-		year := result.Year()
+		// However, due to the way the driver works, the actual upper bound is 2262-04-11 23:47:16.
+		year, month, day := result.Date()
+		if year > 2262 || (year == 2262 && month > 4) || (year == 2262 && month == 4 && day > 11) {
+			return time.Date(2262, time.April, 11, 23, 47, 16, 0, time.UTC), nil
+		}
 		if year < 1900 {
 			return time.Date(1900, time.January, 1, 0, 0, 0, 0, time.UTC), nil
 		}
-		if year > 2299 {
-			return time.Date(2299, time.December, 31, 23, 59, 59, 0, time.UTC), nil
+		hours, minutes, seconds := result.Clock()
+		if year == 2262 && month == 4 && day == 11 && hours == 23 {
+			if minutes > 47 || minutes == 47 && seconds > 16 || minutes == 47 && seconds == 16 {
+				return time.Date(2262, time.April, 11, 23, 47, 16, 0, time.UTC), nil
+			}
 		}
 		return result, nil
 	case pb.DataType_UTC_DATETIME:

destination/db/values/values_test.go

@@ -240,17 +240,35 @@ func TestParseTruncatedDateTime(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, time.Date(1900, 1, 1, 0, 0, 1, 0, time.UTC), val)
 
-	val, err = Parse("test", pb.DataType_NAIVE_DATETIME, "2300-01-01T00:00:00")
+	// year > 2262
+	val, err = Parse("test", pb.DataType_NAIVE_DATETIME, "2263-04-11T00:00:00")
+	assert.NoError(t, err)
+	assert.Equal(t, time.Date(2262, 4, 11, 23, 47, 16, 0, time.UTC), val)
+
+	// year == 2262, month > 04
+	val, err = Parse("test", pb.DataType_NAIVE_DATETIME, "2262-05-11T00:00:00")
 	assert.NoError(t, err)
-	assert.Equal(t, time.Date(2299, 12, 31, 23, 59, 59, 0, time.UTC), val)
+	assert.Equal(t, time.Date(2262, 4, 11, 23, 47, 16, 0, time.UTC), val)
 
-	val, err = Parse("test", pb.DataType_NAIVE_DATETIME, "2299-12-31T23:59:59")
+	// year == 2262, month == 04, day > 11
+	val, err = Parse("test", pb.DataType_NAIVE_DATETIME, "2262-04-12T00:00:00")
 	assert.NoError(t, err)
-	assert.Equal(t, time.Date(2299, 12, 31, 23, 59, 59, 0, time.UTC), val)
+	assert.Equal(t, time.Date(2262, 4, 11, 23, 47, 16, 0, time.UTC), val)
 
-	val, err = Parse("test", pb.DataType_NAIVE_DATETIME, "2299-12-31T23:59:58")
+	// minute > 47
+	val, err = Parse("test", pb.DataType_NAIVE_DATETIME, "2262-04-11T23:48:00")
 	assert.NoError(t, err)
-	assert.Equal(t, time.Date(2299, 12, 31, 23, 59, 58, 0, time.UTC), val)
+	assert.Equal(t, time.Date(2262, 4, 11, 23, 47, 16, 0, time.UTC), val)
+
+	// seconds > 16
+	val, err = Parse("test", pb.DataType_NAIVE_DATETIME, "2262-04-11T23:47:17")
+	assert.NoError(t, err)
+	assert.Equal(t, time.Date(2262, 4, 11, 23, 47, 16, 0, time.UTC), val)
+
+	// an exact fit
+	val, err = Parse("test", pb.DataType_NAIVE_DATETIME, "2262-04-11T23:47:16")
+	assert.NoError(t, err)
+	assert.Equal(t, time.Date(2262, 4, 11, 23, 47, 16, 0, time.UTC), val)
 
 	// MySQL-like edge cases
 	val, err = Parse("test", pb.DataType_NAIVE_DATETIME, "0000-01-01T00:00:00")
@@ -259,7 +277,7 @@ func TestParseTruncatedDateTime(t *testing.T) {
 
 	val, err = Parse("test", pb.DataType_NAIVE_DATETIME, "9999-12-31T23:59:59")
 	assert.NoError(t, err)
-	assert.Equal(t, time.Date(2299, 12, 31, 23, 59, 59, 0, time.UTC), val)
+	assert.Equal(t, time.Date(2262, 4, 11, 23, 47, 16, 0, time.UTC), val)
 }
 
 func TestParseTruncatedUTCDateTime(t *testing.T) {

destination/main_e2e_test.go

@@ -243,6 +243,22 @@ func TestTableNotFound(t *testing.T) {
 	runSDKTestCommand(t, fileName, true) // verify at least no SDK tester errors
 }
 
+func TestTruncateDateValues(t *testing.T) {
+	fileName := "input_truncate_date_values.json"
+	tableName := "truncate_date_values"
+	startServer(t)
+	runSDKTestCommand(t, fileName, true)
+	assertTableRowsWithPK(t, tableName, [][]string{
+		{"1", "1900-01-01", "1900-01-01 00:00:00", "1900-01-01 00:00:00.000000000"},
+		{"2", "2299-12-31", "2262-04-11 23:47:16", "2262-04-11 23:47:16.000000000"}})
+	assertTableColumns(t, tableName, [][]string{
+		{"id", "Int32", ""},
+		{"d", "Nullable(Date32)", ""},
+		{"dt", "Nullable(DateTime64(0, 'UTC'))", ""},
+		{"utc", "Nullable(DateTime64(9, 'UTC'))", ""},
+		{"_fivetran_synced", "DateTime64(9, 'UTC')", ""}})
+}
+
 func TestLargeInputFile(t *testing.T) {
 	t.Skip("Skip large input file test - SDK tester hangs")
 	tableName := "input_large_file"

docs/overview.md

@@ -26,24 +26,32 @@ ClickHouse Cloud.
 
 [Fivetran data types](https://fivetran.com/docs/destinations#datatypes) to ClickHouse mapping overview:
 
-| Fivetran type | ClickHouse type                                                                            |
-|---------------|--------------------------------------------------------------------------------------------|
-| BOOLEAN       | [Bool](https://clickhouse.com/docs/en/sql-reference/data-types/boolean)                    |
-| SHORT         | [Int16](https://clickhouse.com/docs/en/sql-reference/data-types/int-uint)                  |
-| INT           | [Int32](https://clickhouse.com/docs/en/sql-reference/data-types/int-uint)                  |
-| LONG          | [Int64](https://clickhouse.com/docs/en/sql-reference/data-types/int-uint)                  |
-| BIGDECIMAL    | [Decimal(P, S)](https://clickhouse.com/docs/en/sql-reference/data-types/decimal)           |
-| FLOAT         | [Float32](https://clickhouse.com/docs/en/sql-reference/data-types/float)                   |
-| DOUBLE        | [Float64](https://clickhouse.com/docs/en/sql-reference/data-types/float)                   |
-| LOCALDATE     | [Date32](https://clickhouse.com/docs/en/sql-reference/data-types/date32)                   |
-| LOCALDATETIME | [DateTime64(0, 'UTC')](https://clickhouse.com/docs/en/sql-reference/data-types/datetime64) |
-| INSTANT       | [DateTime64(9, 'UTC')](https://clickhouse.com/docs/en/sql-reference/data-types/datetime64) |
-| STRING        | [String](https://clickhouse.com/docs/en/sql-reference/data-types/string)                   |
-| BINARY        | [String](https://clickhouse.com/docs/en/sql-reference/data-types/string) *             |
-| XML           | [String](https://clickhouse.com/docs/en/sql-reference/data-types/string) *             |
-| JSON          | [String](https://clickhouse.com/docs/en/sql-reference/data-types/string) *             |
-
-> * NOTE: The ClickHouse [String](https://clickhouse.com/docs/en/sql-reference/data-types/string) type can be used
+| Fivetran type | ClickHouse type                                                                                  |
+|---------------|--------------------------------------------------------------------------------------------------|
+| BOOLEAN       | [Bool](https://clickhouse.com/docs/en/sql-reference/data-types/boolean)                          |
+| SHORT         | [Int16](https://clickhouse.com/docs/en/sql-reference/data-types/int-uint)                        |
+| INT           | [Int32](https://clickhouse.com/docs/en/sql-reference/data-types/int-uint)                        |
+| LONG          | [Int64](https://clickhouse.com/docs/en/sql-reference/data-types/int-uint)                        |
+| BIGDECIMAL    | [Decimal(P, S)](https://clickhouse.com/docs/en/sql-reference/data-types/decimal)                 |
+| FLOAT         | [Float32](https://clickhouse.com/docs/en/sql-reference/data-types/float)                         |
+| DOUBLE        | [Float64](https://clickhouse.com/docs/en/sql-reference/data-types/float)                         |
+| LOCALDATE     | [Date32](https://clickhouse.com/docs/en/sql-reference/data-types/date32) *                   |
+| LOCALDATETIME | [DateTime64(0, 'UTC')](https://clickhouse.com/docs/en/sql-reference/data-types/datetime64) * |
+| INSTANT       | [DateTime64(9, 'UTC')](https://clickhouse.com/docs/en/sql-reference/data-types/datetime64) * |
+| STRING        | [String](https://clickhouse.com/docs/en/sql-reference/data-types/string)                         |
+| BINARY        | [String](https://clickhouse.com/docs/en/sql-reference/data-types/string) **              |
+| XML           | [String](https://clickhouse.com/docs/en/sql-reference/data-types/string) **              |
+| JSON          | [String](https://clickhouse.com/docs/en/sql-reference/data-types/string) **              |
+
+> * NOTE: the allowed range for `LOCALDATE` values is `[1900-01-01, 2299-12-31]`
+> (see [Date32](https://clickhouse.com/docs/en/sql-reference/data-types/date32));
+> the allowed range for `LOCALDATETIME` and `INSTANT` is `[1900-01-01 00:00:00, 2262-04-11 23:47:16]`
+> (see [DateTime64](https://clickhouse.com/docs/en/sql-reference/data-types/datetime64)). 
+> If a value does not fit into the allowed range, it will be rounded to the nearest valid value.
+> For example: an input `LOCALDATE` value like `0000-01-01` will be stored as `1900-01-01`, 
+> and `9999-01-01` will be stored as `2299-12-31`.
+
+> ** NOTE: The ClickHouse [String](https://clickhouse.com/docs/en/sql-reference/data-types/string) type can be used
 > to represent an arbitrary set of bytes. The ClickHouse destination adds a column comment to the `JSON`, `BINARY`,
 > and `XML` types to indicate the original data type.
 > [JSON](https://clickhouse.com/docs/en/sql-reference/data-types/json) data type is not used as it is marked as

go.mod

@@ -1,35 +1,37 @@
 module fivetran.com/fivetran_sdk
 
-go 1.20
+go 1.21
+
+toolchain go1.21.11
 
 require (
-	github.com/ClickHouse/clickhouse-go/v2 v2.17.1
-	github.com/google/uuid v1.5.0
-	github.com/klauspost/compress v1.16.7
+	github.com/ClickHouse/clickhouse-go/v2 v2.26.0
+	github.com/google/uuid v1.6.0
+	github.com/klauspost/compress v1.17.7
 	github.com/rs/zerolog v1.32.0
-	github.com/shopspring/decimal v1.3.1
-	github.com/stretchr/testify v1.8.4
-	golang.org/x/sync v0.4.0
+	github.com/shopspring/decimal v1.4.0
+	github.com/stretchr/testify v1.9.0
+	golang.org/x/sync v0.6.0
 	google.golang.org/grpc v1.60.1
 	google.golang.org/protobuf v1.33.0
 )
 
 require (
-	github.com/ClickHouse/ch-go v0.58.2 // indirect
-	github.com/andybalholm/brotli v1.0.6 // indirect
+	github.com/ClickHouse/ch-go v0.61.5 // indirect
+	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-faster/city v1.0.1 // indirect
-	github.com/go-faster/errors v0.6.1 // indirect
+	github.com/go-faster/errors v0.7.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
-	github.com/paulmach/orb v0.10.0 // indirect
-	github.com/pierrec/lz4/v4 v4.1.18 // indirect
+	github.com/paulmach/orb v0.11.1 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
-	go.opentelemetry.io/otel v1.19.0 // indirect
-	go.opentelemetry.io/otel/trace v1.19.0 // indirect
+	go.opentelemetry.io/otel v1.26.0 // indirect
+	go.opentelemetry.io/otel/trace v1.26.0 // indirect
 	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect