From 47baaed8636522391657ba67c08a842e143731cf Mon Sep 17 00:00:00 2001 From: taniabogatsch <44262898+taniabogatsch@users.noreply.github.com> Date: Thu, 20 Jun 2024 15:02:18 +0200 Subject: [PATCH] more testing --- appender_test.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++- data_chunk.go | 2 +- errors_test.go | 20 ++++++++++++++++++ types.go | 4 ++++ types_test.go | 30 ++++++-------------------- vector.go | 47 +++++++++++++++++++++++++++++++++++++---- vector_setters.go | 10 ++++----- 7 files changed, 132 insertions(+), 35 deletions(-) diff --git a/appender_test.go b/appender_test.go index 0031740e..16a33fd7 100644 --- a/appender_test.go +++ b/appender_test.go @@ -6,6 +6,7 @@ import ( "database/sql/driver" "encoding/json" "fmt" + "math/big" "math/rand" "testing" "time" @@ -115,7 +116,7 @@ func cleanupAppender[T require.TestingT](t T, c *Connector, con driver.Conn, a * require.NoError(t, c.Close()) } -func prepareAppender(t *testing.T, createTbl string) (*Connector, driver.Conn, *Appender) { +func prepareAppender[T require.TestingT](t T, createTbl string) (*Connector, driver.Conn, *Appender) { c, err := NewConnector("", nil) require.NoError(t, err) @@ -132,12 +133,14 @@ func prepareAppender(t *testing.T, createTbl string) (*Connector, driver.Conn, * } func TestAppenderClose(t *testing.T) { + t.Parallel() c, con, a := prepareAppender(t, `CREATE TABLE test (i INTEGER)`) require.NoError(t, a.AppendRow(int32(42))) cleanupAppender(t, c, con, a) } func TestAppendChunks(t *testing.T) { + t.Parallel() c, con, a := prepareAppender(t, ` CREATE TABLE test ( id BIGINT, @@ -176,6 +179,7 @@ func TestAppendChunks(t *testing.T) { } func TestAppenderList(t *testing.T) { + t.Parallel() c, con, a := prepareAppender(t, ` CREATE TABLE test ( string_list VARCHAR[], @@ -212,6 +216,7 @@ func TestAppenderList(t *testing.T) { } func TestAppenderNested(t *testing.T) { + t.Parallel() c, con, a := prepareAppender(t, ` CREATE TABLE test ( id BIGINT, @@ -352,6 +357,7 @@ func TestAppenderNested(t *testing.T) { } func TestAppenderNullList(t *testing.T) { + t.Parallel() c, con, a := prepareAppender(t, `CREATE TABLE test (int_slice VARCHAR[][][])`) require.NoError(t, a.AppendRow([][][]string{{{}}})) @@ -394,6 +400,7 @@ func TestAppenderNullList(t *testing.T) { } func TestAppenderNullStruct(t *testing.T) { + t.Parallel() c, con, a := prepareAppender(t, ` CREATE TABLE test ( simple_struct STRUCT(A INT, B VARCHAR) @@ -425,6 +432,7 @@ func TestAppenderNullStruct(t *testing.T) { } func TestAppenderNestedNullStruct(t *testing.T) { + t.Parallel() c, con, a := prepareAppender(t, ` CREATE TABLE test ( double_wrapped_struct STRUCT( @@ -477,6 +485,7 @@ func TestAppenderNestedNullStruct(t *testing.T) { } func TestAppenderNullIntAndString(t *testing.T) { + t.Parallel() c, con, a := prepareAppender(t, `CREATE TABLE test (id BIGINT, str VARCHAR)`) require.NoError(t, a.AppendRow(int64(32), "hello")) @@ -517,6 +526,7 @@ func TestAppenderNullIntAndString(t *testing.T) { } func TestAppenderUUID(t *testing.T) { + t.Parallel() c, con, a := prepareAppender(t, `CREATE TABLE test (id UUID)`) id := UUID(uuid.New()) @@ -533,6 +543,7 @@ func TestAppenderUUID(t *testing.T) { } func TestAppenderTsNs(t *testing.T) { + t.Parallel() c, con, a := prepareAppender(t, `CREATE TABLE test (timestamp TIMESTAMP_NS)`) ts := time.Date(2022, time.January, 1, 12, 0, 33, 242, time.UTC) @@ -549,6 +560,7 @@ func TestAppenderTsNs(t *testing.T) { } func TestAppenderDate(t *testing.T) { + t.Parallel() c, con, a := prepareAppender(t, `CREATE TABLE test (date DATE)`) ts := time.Date(1996, time.July, 23, 11, 42, 23, 123, time.UTC) @@ -567,6 +579,7 @@ func TestAppenderDate(t *testing.T) { } func TestAppenderTime(t *testing.T) { + t.Parallel() c, con, a := prepareAppender(t, `CREATE TABLE test (time TIME)`) ts := time.Date(1996, time.July, 23, 11, 42, 23, 123, time.UTC) @@ -583,6 +596,7 @@ func TestAppenderTime(t *testing.T) { } func TestAppenderBlob(t *testing.T) { + t.Parallel() c, con, a := prepareAppender(t, `CREATE TABLE test (data BLOB)`) data := []byte{0x01, 0x02, 0x00, 0x03, 0x04} @@ -611,6 +625,7 @@ func TestAppenderBlob(t *testing.T) { } func TestAppenderBlobTinyInt(t *testing.T) { + t.Parallel() c, con, a := prepareAppender(t, ` CREATE TABLE test ( data UTINYINT[] @@ -649,6 +664,7 @@ func TestAppenderBlobTinyInt(t *testing.T) { } func TestAppenderUint8SliceTinyInt(t *testing.T) { + t.Parallel() c, con, a := prepareAppender(t, ` CREATE TABLE test ( data UTINYINT[] @@ -682,6 +698,41 @@ func TestAppenderUint8SliceTinyInt(t *testing.T) { cleanupAppender(t, c, con, a) } +func TestAppenderDecimal(t *testing.T) { + t.Parallel() + c, con, a := prepareAppender(t, ` + CREATE TABLE test ( + data DECIMAL(4,3) + )`) + + require.NoError(t, a.AppendRow(nil)) + require.NoError(t, a.AppendRow(Decimal{Width: uint8(4), Value: big.NewInt(1), Scale: 3})) + require.NoError(t, a.AppendRow(Decimal{Width: uint8(4), Value: big.NewInt(2), Scale: 3})) + require.NoError(t, a.Flush()) + + // Verify results. + res, err := sql.OpenDB(c).QueryContext(context.Background(), `SELECT CASE WHEN data IS NULL THEN 'NULL' ELSE data::VARCHAR END FROM test`) + require.NoError(t, err) + + expected := []string{ + "NULL", + "0.001", + "0.002", + } + + i := 0 + for res.Next() { + var str string + require.NoError(t, res.Scan(&str)) + require.Equal(t, expected[i], str) + i++ + } + + require.Equal(t, 3, i) + require.NoError(t, res.Close()) + cleanupAppender(t, c, con, a) +} + var jsonInputs = [][]byte{ []byte(`{"c1": 42, "l1": [1, 2, 3], "s1": {"a": 101, "b": ["hello", "world"]}, "l2": [{"a": [{"a": [4.2, 7.9]}]}]}`), []byte(`{"c1": null, "l1": [null, 2, null], "s1": {"a": null, "b": ["hello", null]}, "l2": [{"a": [{"a": [null, 7.9]}]}]}`), @@ -703,6 +754,7 @@ var jsonResults = [][]string{ } func TestAppenderWithJSON(t *testing.T) { + t.Parallel() c, con, a := prepareAppender(t, ` CREATE TABLE test ( c1 UBIGINT, diff --git a/data_chunk.go b/data_chunk.go index 633e4002..8b692496 100644 --- a/data_chunk.go +++ b/data_chunk.go @@ -43,7 +43,7 @@ func (chunk *DataChunk) SetValue(colIdx int, rowIdx int, val any) error { // Ensure that the types match before attempting to set anything. // This is done to prevent failures 'halfway through' writing column values, // potentially corrupting data in that column. - // FIXME: Can we improve efficiency here? We are casting back-and-forth to any A LOT currently. + // FIXME: Can we improve efficiency here? We are casting back-and-forth to any A LOT. // FIXME: Maybe we can make columnar insertions unsafe, i.e., we always assume a correct type. v, err := column.tryCast(val) if err != nil { diff --git a/errors_test.go b/errors_test.go index 576b91cc..dac9d6dd 100644 --- a/errors_test.go +++ b/errors_test.go @@ -173,6 +173,26 @@ func TestErrAppend(t *testing.T) { cleanupAppender(t, c, con, a) } +func TestErrAppendDecimal(t *testing.T) { + c, con, a := prepareAppender(t, `CREATE TABLE test (d DECIMAL(8, 2))`) + + err := a.AppendRow(Decimal{Width: 9, Scale: 2}) + testError(t, err, errAppenderAppendRow.Error(), castErrMsg) + err = a.AppendRow(Decimal{Width: 8, Scale: 3}) + testError(t, err, errAppenderAppendRow.Error(), castErrMsg) + + cleanupAppender(t, c, con, a) +} + +func TestErrAppendEnum(t *testing.T) { + c, con, a := prepareAppender(t, testTypesEnumSQL+";"+`CREATE TABLE test (e my_enum)`) + + err := a.AppendRow("3") + testError(t, err, errAppenderAppendRow.Error(), castErrMsg) + + cleanupAppender(t, c, con, a) +} + func TestErrAppendSimpleStruct(t *testing.T) { c, con, a := prepareAppender(t, ` CREATE TABLE test ( diff --git a/types.go b/types.go index d6a04bfd..b698d350 100644 --- a/types.go +++ b/types.go @@ -176,3 +176,7 @@ func (d *Decimal) Float64() float64 { f, _ := value.Float64() return f } + +func (d *Decimal) toString() string { + return fmt.Sprintf("DECIMAL(%d,%d)", d.Width, d.Scale) +} diff --git a/types_test.go b/types_test.go index 799f00e6..8c24e58c 100644 --- a/types_test.go +++ b/types_test.go @@ -3,7 +3,6 @@ package duckdb import ( "context" "database/sql" - "database/sql/driver" "fmt" "math/big" "strconv" @@ -55,7 +54,7 @@ type testTypesRow struct { Timestamp_tz_col time.Time } -const testTypesTableSQL = `CREATE TABLE types_tbl ( +const testTypesTableSQL = `CREATE TABLE test ( Boolean_col BOOLEAN, Tinyint_col TINYINT, Smallint_col SMALLINT, @@ -158,26 +157,8 @@ func testTypesGenerateRows[T require.TestingT](t T, rowCount int) []testTypesRow return expectedRows } -func testTypesSetup[T require.TestingT](t T) (*Connector, driver.Conn, *Appender) { - c, err := NewConnector("", nil) - require.NoError(t, err) - - _, err = sql.OpenDB(c).Exec(testTypesEnumSQL) - require.NoError(t, err) - - _, err = sql.OpenDB(c).Exec(testTypesTableSQL) - require.NoError(t, err) - - con, err := c.Connect(context.Background()) - require.NoError(t, err) - - a, err := NewAppenderFromConn(con, "", "types_tbl") - require.NoError(t, err) - return c, con, a -} - func testTypesReset[T require.TestingT](t T, c *Connector) { - _, err := sql.OpenDB(c).ExecContext(context.Background(), `DELETE FROM types_tbl`) + _, err := sql.OpenDB(c).ExecContext(context.Background(), `DELETE FROM test`) require.NoError(t, err) } @@ -216,7 +197,7 @@ func testTypes[T require.TestingT](t T, c *Connector, a *Appender, expectedRows } require.NoError(t, a.Flush()) - res, err := sql.OpenDB(c).QueryContext(context.Background(), `SELECT * FROM types_tbl ORDER BY Smallint_col`) + res, err := sql.OpenDB(c).QueryContext(context.Background(), `SELECT * FROM test ORDER BY Smallint_col`) require.NoError(t, err) // Scan the rows. @@ -260,8 +241,9 @@ func testTypes[T require.TestingT](t T, c *Connector, a *Appender, expectedRows } func TestTypes(t *testing.T) { + t.Parallel() expectedRows := testTypesGenerateRows(t, 3) - c, con, a := testTypesSetup(t) + c, con, a := prepareAppender(t, testTypesEnumSQL+";"+testTypesTableSQL) actualRows := testTypes(t, c, a, expectedRows) for i := range actualRows { @@ -278,7 +260,7 @@ func TestTypes(t *testing.T) { func BenchmarkTypes(b *testing.B) { expectedRows := testTypesGenerateRows(b, GetDataChunkCapacity()*3+10) - c, con, a := testTypesSetup(b) + c, con, a := prepareAppender(b, testTypesEnumSQL+";"+testTypesTableSQL) for n := 0; n < b.N; n++ { _ = testTypes(b, c, a, expectedRows) diff --git a/vector.go b/vector.go index f901580a..7094c118 100644 --- a/vector.go +++ b/vector.go @@ -22,12 +22,17 @@ type vector struct { setFn fnSetVectorValue // The data type of the vector. duckdbType C.duckdb_type - // The child names of STRUCT vectors. - childNames []string // The child vectors of nested data types. childVectors []vector + + // The child names of STRUCT vectors. + childNames []string // The dictionary for ENUM types. dict map[string]uint32 + // The width of DECIMAL types. + width uint8 + // The scale of DECIMAL types. + scale uint8 } func (vec *vector) tryCast(val any) (any, error) { @@ -70,12 +75,14 @@ func (vec *vector) tryCast(val any) (any, error) { return tryPrimitiveCast[*big.Int](val, reflect.TypeOf(big.Int{}).String()) case C.DUCKDB_TYPE_UHUGEINT: return nil, unsupportedTypeError(duckdbTypeMap[vec.duckdbType]) - case C.DUCKDB_TYPE_VARCHAR, C.DUCKDB_TYPE_ENUM: + case C.DUCKDB_TYPE_VARCHAR: return tryPrimitiveCast[string](val, reflect.String.String()) case C.DUCKDB_TYPE_BLOB: return tryPrimitiveCast[[]byte](val, reflect.TypeOf([]byte{}).String()) case C.DUCKDB_TYPE_DECIMAL: - return tryPrimitiveCast[Decimal](val, reflect.TypeOf(Decimal{}).String()) + return vec.tryCastDecimal(val) + case C.DUCKDB_TYPE_ENUM: + return vec.tryCastEnum(val) case C.DUCKDB_TYPE_LIST: return vec.tryCastList(val) case C.DUCKDB_TYPE_STRUCT: @@ -133,6 +140,34 @@ func tryNumericCast[T numericType](val any, expected string) (T, error) { return v, castError(goType.String(), expected) } +func (vec *vector) tryCastDecimal(val any) (Decimal, error) { + v, ok := val.(Decimal) + if !ok { + goType := reflect.TypeOf(val) + return v, castError(goType.String(), reflect.TypeOf(Decimal{}).String()) + } + + if v.Width != vec.width || v.Scale != vec.scale { + d := Decimal{Width: vec.width, Scale: vec.scale} + return v, castError(d.toString(), v.toString()) + } + return v, nil +} + +func (vec *vector) tryCastEnum(val any) (string, error) { + v, ok := val.(string) + if !ok { + goType := reflect.TypeOf(val) + return v, castError(goType.String(), reflect.String.String()) + } + + _, ok = vec.dict[v] + if !ok { + return v, castError(v, "ENUM value") + } + return v, nil +} + func (vec *vector) tryCastList(val any) ([]any, error) { goType := reflect.TypeOf(val) if goType.Kind() != reflect.Slice { @@ -369,6 +404,10 @@ func (vec *vector) initCString(duckdbType C.duckdb_type) { } func (vec *vector) initDecimal(logicalType C.duckdb_logical_type, colIdx int) error { + // Get the width and scale. + vec.width = uint8(C.duckdb_decimal_width(logicalType)) + vec.scale = uint8(C.duckdb_decimal_scale(logicalType)) + internalType := C.duckdb_decimal_internal_type(logicalType) switch internalType { case C.DUCKDB_TYPE_SMALLINT, C.DUCKDB_TYPE_INTEGER, C.DUCKDB_TYPE_BIGINT, C.DUCKDB_TYPE_HUGEINT: diff --git a/vector_setters.go b/vector_setters.go index dd0e8eaa..440c1bd5 100644 --- a/vector_setters.go +++ b/vector_setters.go @@ -106,17 +106,17 @@ func (vec *vector) setCString(rowIdx C.idx_t, val any) { } func (vec *vector) setDecimal(internalType C.duckdb_type, rowIdx C.idx_t, val any) { - v := val.(*big.Int) + v := val.(Decimal) switch internalType { case C.DUCKDB_TYPE_SMALLINT: - setPrimitive(vec, rowIdx, int16(v.Int64())) + setPrimitive(vec, rowIdx, int16(v.Value.Int64())) case C.DUCKDB_TYPE_INTEGER: - setPrimitive(vec, rowIdx, int32(v.Int64())) + setPrimitive(vec, rowIdx, int32(v.Value.Int64())) case C.DUCKDB_TYPE_BIGINT: - setPrimitive(vec, rowIdx, v.Int64()) + setPrimitive(vec, rowIdx, v.Value.Int64()) case C.DUCKDB_TYPE_HUGEINT: - value, _ := hugeIntFromNative(v) + value, _ := hugeIntFromNative(v.Value) setPrimitive(vec, rowIdx, value) } }