Small fixes, linter changes, and improved casts

appender.go

@@ -176,10 +176,17 @@ func (a *Appender) appendDataChunks() error {
 	var state C.duckdb_state
 	var err error
 
-	for _, chunk := range a.chunks {
-		if err = chunk.setSize(); err != nil {
+	for i, chunk := range a.chunks {
+
+		// All data chunks except the last are at maximum capacity.
+		size := chunk.GetCapacity()
+		if i == len(a.chunks)-1 {
+			size = a.rowCount
+		}
+		if err = chunk.SetSize(size); err != nil {
 			break
 		}
+
 		state = C.duckdb_append_data_chunk(a.duckdbAppender, chunk.data)
 		if state == C.DuckDBError {
 			err = duckdbError(C.duckdb_appender_error(a.duckdbAppender))
@@ -198,3 +205,20 @@ func (a *Appender) closeDataChunks() {
 	}
 	a.chunks = a.chunks[:0]
 }
+
+func mallocTypeSlice(count int) (unsafe.Pointer, []C.duckdb_logical_type) {
+	var dummy C.duckdb_logical_type
+	size := C.size_t(unsafe.Sizeof(dummy))
+
+	ptr := unsafe.Pointer(C.malloc(C.size_t(count) * size))
+	slice := (*[1 << 30]C.duckdb_logical_type)(ptr)[:count:count]
+
+	return ptr, slice
+}
+
+func destroyTypeSlice(ptr unsafe.Pointer, slice []C.duckdb_logical_type) {
+	for _, t := range slice {
+		C.duckdb_destroy_logical_type(&t)
+	}
+	C.free(ptr)
+}

data_chunk.go

@@ -18,6 +18,20 @@ type DataChunk struct {
 	columns []vector
 }
 
+// GetCapacity returns the capacity of a data chunk.
+func (chunk *DataChunk) GetCapacity() int {
+	return int(C.duckdb_vector_size())
+}
+
+// SetSize sets the internal size of the data chunk. Cannot exceed GetCapacity().
+func (chunk *DataChunk) SetSize(size int) error {
+	if size > chunk.GetCapacity() {
+		return getError(errAPI, errVectorSize)
+	}
+	C.duckdb_data_chunk_set_size(chunk.data, C.idx_t(size))
+	return nil
+}
+
 // SetValue writes a single value to a column in a data chunk. Note that this requires casting the type for each invocation.
 func (chunk *DataChunk) SetValue(colIdx int, rowIdx int, val any) error {
 	if colIdx >= len(chunk.columns) {
@@ -26,6 +40,8 @@ func (chunk *DataChunk) SetValue(colIdx int, rowIdx int, val any) error {
 	column := &chunk.columns[colIdx]
 
 	// Ensure that the types match before attempting to set anything.
+	// This is done to prevent failures 'halfway through' writing column values,
+	// potentially corrupting data in that column.
 	v, err := column.tryCast(val)
 	if err != nil {
 		return columnError(err, colIdx)
@@ -67,27 +83,3 @@ func (chunk *DataChunk) initFromTypes(ptr unsafe.Pointer, types []C.duckdb_logic
 func (chunk *DataChunk) close() {
 	C.duckdb_destroy_data_chunk(&chunk.data)
 }
-
-func (chunk *DataChunk) setSize() error {
-	if len(chunk.columns) == 0 {
-		C.duckdb_data_chunk_set_size(chunk.data, C.idx_t(0))
-		return nil
-	}
-
-	allEqual := true
-	maxSize := C.idx_t(chunk.columns[0].size)
-	for i := 0; i < len(chunk.columns); i++ {
-		if chunk.columns[i].size != maxSize {
-			allEqual = false
-		}
-		if chunk.columns[i].size > maxSize {
-			maxSize = chunk.columns[i].size
-		}
-	}
-
-	if !allEqual {
-		return errDriver
-	}
-	C.duckdb_data_chunk_set_size(chunk.data, maxSize)
-	return nil
-}

errors.go

@@ -57,7 +57,9 @@ const (
 
 var (
 	errDriver = errors.New("internal driver error: please file a bug report")
-	errAPI    = errors.New("API error")
+
+	errAPI        = errors.New("API error")
+	errVectorSize = errors.New("data chunks cannot exceed duckdb's internal vector size")
 
 	errParseDSN  = errors.New("could not parse DSN for database")
 	errOpen      = errors.New("could not open database")

replacement_scan.go

@@ -8,6 +8,7 @@ package duckdb
 	void replacement_scan_destroy_data(void *);
 */
 import "C"
+
 import (
 	"runtime/cgo"
 	"unsafe"

replacement_scan_test.go

@@ -3,20 +3,20 @@ package duckdb
 import (
 	"database/sql"
 	"database/sql/driver"
-	"github.com/stretchr/testify/require"
 	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 func TestReplacementScan(t *testing.T) {
-
 	connector, err := NewConnector("", func(execer driver.ExecerContext) error {
 		return nil
 	})
 
 	require.NoError(t, err)
 	defer connector.Close()
 
-	var rangeRows = 100
+	rangeRows := 100
 	RegisterReplacementScan(connector, func(tableName string) (string, []any, error) {
 		return "range", []any{int64(rangeRows)}, nil
 	})
@@ -40,5 +40,4 @@ func TestReplacementScan(t *testing.T) {
 	if rangeRows != 0 {
 		require.Fail(t, "expected 0, got %d", rangeRows)
 	}
-
 }

vector.go

@@ -25,8 +25,6 @@ type vector struct {
 	childNames []string
 	// The child vectors of nested data types.
 	childVectors []vector
-	// The number of values in this vector.
-	size C.idx_t
 }
 
 func (vec *vector) tryCast(val any) (any, error) {
@@ -84,29 +82,31 @@ func (*vector) canNil(val reflect.Value) bool {
 	return false
 }
 
-func tryPrimitiveCast[T any](val any, expected string) (any, error) {
-	if v, ok := val.(T); ok {
+func tryPrimitiveCast[T any](val any, expected string) (T, error) {
+	v, ok := val.(T)
+	if ok {
 		return v, nil
 	}
 
 	goType := reflect.TypeOf(val)
-	return nil, castError(goType.String(), expected)
+	return v, castError(goType.String(), expected)
 }
 
-func tryNumericCast[T numericType](val any, expected string) (any, error) {
-	if v, ok := val.(T); ok {
+func tryNumericCast[T numericType](val any, expected string) (T, error) {
+	v, ok := val.(T)
+	if ok {
 		return v, nil
 	}
 
 	// JSON unmarshalling uses float64 for numbers.
 	// We might want to add more implicit casts here.
-	switch v := val.(type) {
+	switch value := val.(type) {
 	case float64:
-		return convertNumericType[float64, T](v), nil
+		return convertNumericType[float64, T](value), nil
 	}
 
 	goType := reflect.TypeOf(val)
-	return nil, castError(goType.String(), expected)
+	return v, castError(goType.String(), expected)
 }
 
 func (vec *vector) tryCastList(val any) ([]any, error) {
@@ -269,19 +269,17 @@ func (vec *vector) getChildVectors(vector C.duckdb_vector) {
 
 func initPrimitive[T any](vec *vector, duckdbType C.duckdb_type) {
 	vec.setFn = func(vec *vector, rowIdx C.idx_t, val any) {
-		vec.size++
 		if val == nil {
 			vec.setNull(rowIdx)
 			return
 		}
-		setPrimitive[T](vec, rowIdx, val)
+		setPrimitive[T](vec, rowIdx, val.(T))
 	}
 	vec.duckdbType = duckdbType
 }
 
 func (vec *vector) initTS(duckdbType C.duckdb_type) {
 	vec.setFn = func(vec *vector, rowIdx C.idx_t, val any) {
-		vec.size++
 		if val == nil {
 			vec.setNull(rowIdx)
 			return
@@ -293,7 +291,6 @@ func (vec *vector) initTS(duckdbType C.duckdb_type) {
 
 func (vec *vector) initDate() {
 	vec.setFn = func(vec *vector, rowIdx C.idx_t, val any) {
-		vec.size++
 		if val == nil {
 			vec.setNull(rowIdx)
 			return
@@ -305,7 +302,6 @@ func (vec *vector) initDate() {
 
 func (vec *vector) initCString(duckdbType C.duckdb_type) {
 	vec.setFn = func(vec *vector, rowIdx C.idx_t, val any) {
-		vec.size++
 		if val == nil {
 			vec.setNull(rowIdx)
 			return
@@ -328,7 +324,6 @@ func (vec *vector) initList(logicalType C.duckdb_logical_type, colIdx int) error
 	}
 
 	vec.setFn = func(vec *vector, rowIdx C.idx_t, val any) {
-		vec.size++
 		if val == nil {
 			vec.setNull(rowIdx)
 			return
@@ -363,7 +358,6 @@ func (vec *vector) initStruct(logicalType C.duckdb_logical_type, colIdx int) err
 	}
 
 	vec.setFn = func(vec *vector, rowIdx C.idx_t, val any) {
-		vec.size++
 		if val == nil {
 			vec.setNull(rowIdx)
 			return
@@ -376,7 +370,6 @@ func (vec *vector) initStruct(logicalType C.duckdb_logical_type, colIdx int) err
 
 func (vec *vector) initUUID() {
 	vec.setFn = func(vec *vector, rowIdx C.idx_t, val any) {
-		vec.size++
 		if val == nil {
 			vec.setNull(rowIdx)
 			return

vector_setters.go

@@ -11,6 +11,9 @@ import (
 	"unsafe"
 )
 
+// secondsPerDay to calculate the days since 1970-01-01.
+const secondsPerDay = 24 * 60 * 60
+
 // fnSetVectorValue is the setter callback function for any (nested) vector.
 type fnSetVectorValue func(vec *vector, rowIdx C.idx_t, val any)
 
@@ -26,10 +29,10 @@ func (vec *vector) setNull(rowIdx C.idx_t) {
 	}
 }
 
-func setPrimitive[T any](vec *vector, rowIdx C.idx_t, val any) {
+func setPrimitive[T any](vec *vector, rowIdx C.idx_t, v T) {
 	ptr := C.duckdb_vector_get_data(vec.duckdbVector)
 	xs := (*[1 << 31]T)(ptr)
-	xs[rowIdx] = val.(T)
+	xs[rowIdx] = v
 }
 
 func (vec *vector) setTS(duckdbType C.duckdb_type, rowIdx C.idx_t, val any) {