From 31fc9def4493310b7e5516157805e0f88a17f44d Mon Sep 17 00:00:00 2001 From: taniabogatsch <44262898+taniabogatsch@users.noreply.github.com> Date: Mon, 23 Sep 2024 14:42:14 +0200 Subject: [PATCH] statically link JSON --- Makefile | 9 ++++- README.md | 104 +++++++++++++++++++++++-------------------------- duckdb_test.go | 48 ++--------------------- 3 files changed, 59 insertions(+), 102 deletions(-) diff --git a/Makefile b/Makefile index 90a522a0..5bc9166b 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ examples: .PHONY: test test: - go test -v -race -count=1 . + go test -v -count=1 . .PHONY: deps.header deps.header: @@ -24,11 +24,12 @@ duckdb: rm -rf duckdb git clone -b ${DUCKDB_BRANCH} --depth 1 ${DUCKDB_REPO} -DUCKDB_COMMON_BUILD_FLAGS := BUILD_SHELL=0 BUILD_UNITTESTS=0 DUCKDB_PLATFORM=any +DUCKDB_COMMON_BUILD_FLAGS := BUILD_SHELL=0 BUILD_UNITTESTS=0 DUCKDB_PLATFORM=any ENABLE_EXTENSION_AUTOLOADING=1 ENABLE_EXTENSION_AUTOINSTALL=1 BUILD_EXTENSIONS="json" .PHONY: deps.darwin.amd64 deps.darwin.amd64: duckdb if [ "$(shell uname -s | tr '[:upper:]' '[:lower:]')" != "darwin" ]; then echo "Error: must run build on darwin"; false; fi + mkdir -p deps/darwin_amd64 cd duckdb && \ CFLAGS="-target x86_64-apple-macos11 -O3" CXXFLAGS="-target x86_64-apple-macos11 -O3" ${DUCKDB_COMMON_BUILD_FLAGS} make bundle-library -j 2 @@ -37,6 +38,7 @@ deps.darwin.amd64: duckdb .PHONY: deps.darwin.arm64 deps.darwin.arm64: duckdb if [ "$(shell uname -s | tr '[:upper:]' '[:lower:]')" != "darwin" ]; then echo "Error: must run build on darwin"; false; fi + mkdir -p deps/darwin_arm64 cd duckdb && \ CFLAGS="-target arm64-apple-macos11 -O3" CXXFLAGS="-target arm64-apple-macos11 -O3" ${DUCKDB_COMMON_BUILD_FLAGS} make bundle-library -j 2 @@ -45,6 +47,7 @@ deps.darwin.arm64: duckdb .PHONY: deps.linux.amd64 deps.linux.amd64: duckdb if [ "$(shell uname -s | tr '[:upper:]' '[:lower:]')" != "linux" ]; then echo "Error: must run build on linux"; false; fi + mkdir -p deps/linux_amd64 cd duckdb && \ CFLAGS="-O3" CXXFLAGS="-O3" ${DUCKDB_COMMON_BUILD_FLAGS} make bundle-library -j 2 @@ -53,6 +56,7 @@ deps.linux.amd64: duckdb .PHONY: deps.linux.arm64 deps.linux.arm64: duckdb if [ "$(shell uname -s | tr '[:upper:]' '[:lower:]')" != "linux" ]; then echo "Error: must run build on linux"; false; fi + mkdir -p deps/linux_arm64 cd duckdb && \ CC="aarch64-linux-gnu-gcc" CXX="aarch64-linux-gnu-g++" CFLAGS="-O3" CXXFLAGS="-O3" ${DUCKDB_COMMON_BUILD_FLAGS} make bundle-library -j 2 @@ -61,6 +65,7 @@ deps.linux.arm64: duckdb .PHONY: deps.freebsd.amd64 deps.freebsd.amd64: duckdb if [ "$(shell uname -s | tr '[:upper:]' '[:lower:]')" != "freebsd" ]; then echo "Error: must run build on freebsd"; false; fi + mkdir -p deps/freebsd_amd64 cd duckdb && \ CFLAGS="-O3" CXXFLAGS="-O3" ${DUCKDB_COMMON_BUILD_FLAGS} gmake bundle-library -j 2 diff --git a/README.md b/README.md index f459a07c..da7cb4b2 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,7 @@ go get github.com/marcboeker/go-duckdb ```go db, err := sql.Open("duckdb", "") -if err != nil { - ... -} +check(err) defer db.Close() ``` @@ -29,9 +27,7 @@ the file does not exist, then DuckDB creates it. ```go db, err := sql.Open("duckdb", "/path/to/foo.db") -if err != nil { - ... -} +check(err) defer db.Close() ``` @@ -39,33 +35,30 @@ If you want to set specific [config options for DuckDB](https://duckdb.org/docs/ ```go db, err := sql.Open("duckdb", "/path/to/foo.db?access_mode=read_only&threads=4") -if err != nil { - ... -} +check(err) defer db.Close() ``` -Alternatively, you can use [sql.OpenDB](https://cs.opensource.google/go/go/+/refs/tags/go1.23.0:src/database/sql/sql.go;l=824). That way, you can perform initialization steps in a callback function before opening the database. -Here's an example that installs and loads the JSON extension when opening a database with `sql.OpenDB(connector)`. +Alternatively, you can use [sql.OpenDB](https://cs.opensource.google/go/go/+/refs/tags/go1.23.0:src/database/sql/sql.go;l=824). +That way, you can perform initialization steps in a callback function before opening the database. +Here's an example that configures some database parameters when opening a database with `sql.OpenDB(connector)`. ```go connector, err := duckdb.NewConnector("/path/to/foo.db?access_mode=read_only&threads=4", func(execer driver.ExecerContext) error { bootQueries := []string{ - "INSTALL 'json'", - "LOAD 'json'", + "SET schema=main", + "SET search_path=main", } for _, query := range bootQueries { _, err = execer.ExecContext(context.Background(), query, nil) if err != nil { - ... + return err } } return nil }) -if err != nil { - ... -} +check(err) db := sql.OpenDB(connector) defer db.Close() @@ -87,46 +80,39 @@ conn, err := db.Conn(context.Background()) defer conn.Close() rows, err := conn.QueryContext(context.Background(), "SELECT 42") -// alternatively, rows.Next() has to return false +// Alternatively, rows.Next() has to return false. rows.Close() appender, err := NewAppenderFromConn(conn, "", "test") defer appender.Close() -// if not passed to sql.OpenDB +// If not passed to sql.OpenDB. connector, err := NewConnector("", nil) defer connector.Close() ``` ## DuckDB Appender API -If you want to use the [DuckDB Appender API](https://duckdb.org/docs/data/appender.html), you can obtain a new `Appender` by passing a DuckDB connection to `NewAppenderFromConn()`. See `examples/appender.go` for a complete example. +If you want to use the [DuckDB Appender API](https://duckdb.org/docs/data/appender.html), you can obtain a new `Appender` by passing a DuckDB connection to `NewAppenderFromConn()`. +See `examples/appender.go` for a complete example. ```go connector, err := duckdb.NewConnector("test.db", nil) -if err != nil { - ... -} +check(err) defer connector.Close() conn, err := connector.Connect(context.Background()) -if err != nil { - ... -} +check(err) defer conn.Close() -// obtain an appender from the connection -// NOTE: the table 'test_tbl' must exist in test.db +// Obtain an appender from the connection. +// NOTE: The table 'test_tbl' must exist in test.db. appender, err := NewAppenderFromConn(conn, "", "test_tbl") -if err != nil { - ... -} +check(err) defer appender.Close() err = appender.AppendRow(...) -if err != nil { - ... -} +check(err) ``` ## DuckDB Apache Arrow Interface @@ -135,35 +121,29 @@ If you want to use the [DuckDB Arrow Interface](https://duckdb.org/docs/api/c/ap ```go connector, err := duckdb.NewConnector("", nil) -if err != nil { - ... -} +check(err) defer connector.Close() conn, err := connector.Connect(context.Background()) -if err != nil { - ... -} +check(err) defer conn.Close() -// obtain the Arrow from the connection +// Obtain the Arrow from the connection. arrow, err := duckdb.NewArrowFromConn(conn) -if err != nil { - ... -} +check(err) rdr, err := arrow.QueryContext(context.Background(), "SELECT * FROM generate_series(1, 10)") -if err != nil { - ... -} +check(err) defer rdr.Release() for rdr.Next() { - // process records + // Process each record. } ``` -The Arrow interface is a heavy dependency. If you do not need it, you can disable it by passing `-tags=no_duckdb_arrow` to `go build`. This will be made opt-in in V2. +The Arrow interface is a heavy dependency. +If you do not need it, you can disable it by passing `-tags=no_duckdb_arrow` to `go build`. +This will be made opt-in in V2. ```sh go build -tags="no_duckdb_arrow" @@ -182,26 +162,40 @@ Now you can build your module as usual. ## Linking DuckDB -By default, `go-duckdb` statically links DuckDB into your binary. Statically linking DuckDB adds around 30 MB to your binary size. On Linux (Intel) and macOS (Intel and ARM), `go-duckdb` bundles pre-compiled static libraries for fast builds. +By default, `go-duckdb` statically links DuckDB into your binary. +Statically linking DuckDB increases your binary size. +`go-duckdb` bundles pre-compiled static libraries for some OS and architecture combinations. +- MacOS: amd64, arm64. +- Linux: amd64, arm64. +- FreeBSD: amd64. -Alternatively, you can dynamically link DuckDB by passing `-tags=duckdb_use_lib` to `go build`. You must have a copy of `libduckdb` available on your system (`.so` on Linux or `.dylib` on macOS), which you can download from the DuckDB [releases page](https://github.com/duckdb/duckdb/releases). For example: +Alternatively, you can dynamically link DuckDB by passing `-tags=duckdb_use_lib` to `go build`. +You must have a copy of `libduckdb` available on your system (`.so` on Linux or `.dylib` on macOS), which you can download from the DuckDB [releases page](https://github.com/duckdb/duckdb/releases). +For example: ```sh -# On Linux +# On Linux. CGO_ENABLED=1 CGO_LDFLAGS="-L/path/to/libs" go build -tags=duckdb_use_lib main.go LD_LIBRARY_PATH=/path/to/libs ./main -# On macOS +# On macOS. CGO_ENABLED=1 CGO_LDFLAGS="-L/path/to/libs" go build -tags=duckdb_use_lib main.go DYLD_LIBRARY_PATH=/path/to/libs ./main ``` +## DuckDB Extensions + +`go-duckdb` statically builds the `JSON` extension for its pre-compiled libraries. +Additionally, automatic extension loading is enabled. +The extensions available differ between the pre-compiled libraries. +Thus, if you fail to install and load an extension, you might have to link a custom DuckDB. + ## Notes -`TIMESTAMP vs. TIMESTAMP_TZ` +**`TIMESTAMP vs. TIMESTAMP_TZ`** In the C API, DuckDB stores both `TIMESTAMP` and `TIMESTAMP_TZ` as `duckdb_timestamp`, which holds the number of microseconds elapsed since January 1, 1970 UTC (i.e., an instant without offset information). When passing a `time.Time` to go-duckdb, go-duckdb transforms it to an instant with `UnixMicro()`, even when using `TIMESTAMP_TZ`. Later, scanning either type of value returns an instant, as SQL types do not model -time zone information for individual values. +time zone information for individual values. \ No newline at end of file diff --git a/duckdb_test.go b/duckdb_test.go index 44d72662..500c862b 100644 --- a/duckdb_test.go +++ b/duckdb_test.go @@ -51,27 +51,6 @@ func TestOpen(t *testing.T) { } func TestConnectorBootQueries(t *testing.T) { - t.Run("many boot queries", func(t *testing.T) { - connector, err := NewConnector("", func(execer driver.ExecerContext) error { - bootQueries := []string{ - "INSTALL 'json'", - "LOAD 'json'", - "SET schema=main", - "SET search_path=main", - } - - for _, query := range bootQueries { - _, err := execer.ExecContext(context.Background(), query, nil) - require.NoError(t, err) - } - return nil - }) - require.NoError(t, err) - - db := sql.OpenDB(connector) - defer db.Close() - }) - t.Run("readme example", func(t *testing.T) { db, err := sql.Open("duckdb", "foo.db") require.NoError(t, err) @@ -79,12 +58,11 @@ func TestConnectorBootQueries(t *testing.T) { connector, err := NewConnector("foo.db?access_mode=read_only&threads=4", func(execer driver.ExecerContext) error { bootQueries := []string{ - "INSTALL 'json'", - "LOAD 'json'", + "SET schema=main", + "SET search_path=main", } - for _, query := range bootQueries { - _, err := execer.ExecContext(context.Background(), query, nil) + _, err = execer.ExecContext(context.Background(), query, nil) require.NoError(t, err) } return nil @@ -148,17 +126,6 @@ func TestConnPool(t *testing.T) { func TestConnInit(t *testing.T) { connector, err := NewConnector("", func(execer driver.ExecerContext) error { - bootQueries := []string{ - "INSTALL 'json'", - "LOAD 'json'", - } - - for _, qry := range bootQueries { - _, err := execer.ExecContext(context.Background(), qry, nil) - if err != nil { - return err - } - } return nil }) require.NoError(t, err) @@ -280,8 +247,6 @@ func TestQuery(t *testing.T) { func TestJSON(t *testing.T) { t.Parallel() db := openDB(t) - - loadJSONExt(t, db) var data string t.Run("select empty JSON", func(t *testing.T) { @@ -671,13 +636,6 @@ func openDB(t *testing.T) *sql.DB { return db } -func loadJSONExt(t *testing.T, db *sql.DB) { - _, err := db.Exec("INSTALL 'json'") - require.NoError(t, err) - _, err = db.Exec("LOAD 'json'") - require.NoError(t, err) -} - func createTable(db *sql.DB, t *testing.T, sql string) *sql.Result { res, err := db.Exec(sql) require.NoError(t, err)