Skip to content

Commit

Permalink
statically link JSON
Browse files Browse the repository at this point in the history
  • Loading branch information
@taniabogatsch
taniabogatsch committed Sep 23, 2024
1 parent 80041da commit 31fc9de
Show file tree
Hide file tree
Showing 3 changed files with 59 additions and 102 deletions.
9 changes: 7 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ examples:

.PHONY: test
test:
go test -v -race -count=1 .
go test -v -count=1 .

.PHONY: deps.header
deps.header:
Expand All @@ -24,11 +24,12 @@ duckdb:
rm -rf duckdb
git clone -b ${DUCKDB_BRANCH} --depth 1 ${DUCKDB_REPO}

DUCKDB_COMMON_BUILD_FLAGS := BUILD_SHELL=0 BUILD_UNITTESTS=0 DUCKDB_PLATFORM=any
DUCKDB_COMMON_BUILD_FLAGS := BUILD_SHELL=0 BUILD_UNITTESTS=0 DUCKDB_PLATFORM=any ENABLE_EXTENSION_AUTOLOADING=1 ENABLE_EXTENSION_AUTOINSTALL=1 BUILD_EXTENSIONS="json"

.PHONY: deps.darwin.amd64
deps.darwin.amd64: duckdb
if [ "$(shell uname -s | tr '[:upper:]' '[:lower:]')" != "darwin" ]; then echo "Error: must run build on darwin"; false; fi
mkdir -p deps/darwin_amd64

cd duckdb && \
CFLAGS="-target x86_64-apple-macos11 -O3" CXXFLAGS="-target x86_64-apple-macos11 -O3" ${DUCKDB_COMMON_BUILD_FLAGS} make bundle-library -j 2
Expand All @@ -37,6 +38,7 @@ deps.darwin.amd64: duckdb
.PHONY: deps.darwin.arm64
deps.darwin.arm64: duckdb
if [ "$(shell uname -s | tr '[:upper:]' '[:lower:]')" != "darwin" ]; then echo "Error: must run build on darwin"; false; fi
mkdir -p deps/darwin_arm64

cd duckdb && \
CFLAGS="-target arm64-apple-macos11 -O3" CXXFLAGS="-target arm64-apple-macos11 -O3" ${DUCKDB_COMMON_BUILD_FLAGS} make bundle-library -j 2
Expand All @@ -45,6 +47,7 @@ deps.darwin.arm64: duckdb
.PHONY: deps.linux.amd64
deps.linux.amd64: duckdb
if [ "$(shell uname -s | tr '[:upper:]' '[:lower:]')" != "linux" ]; then echo "Error: must run build on linux"; false; fi
mkdir -p deps/linux_amd64

cd duckdb && \
CFLAGS="-O3" CXXFLAGS="-O3" ${DUCKDB_COMMON_BUILD_FLAGS} make bundle-library -j 2
Expand All @@ -53,6 +56,7 @@ deps.linux.amd64: duckdb
.PHONY: deps.linux.arm64
deps.linux.arm64: duckdb
if [ "$(shell uname -s | tr '[:upper:]' '[:lower:]')" != "linux" ]; then echo "Error: must run build on linux"; false; fi
mkdir -p deps/linux_arm64

cd duckdb && \
CC="aarch64-linux-gnu-gcc" CXX="aarch64-linux-gnu-g++" CFLAGS="-O3" CXXFLAGS="-O3" ${DUCKDB_COMMON_BUILD_FLAGS} make bundle-library -j 2
Expand All @@ -61,6 +65,7 @@ deps.linux.arm64: duckdb
.PHONY: deps.freebsd.amd64
deps.freebsd.amd64: duckdb
if [ "$(shell uname -s | tr '[:upper:]' '[:lower:]')" != "freebsd" ]; then echo "Error: must run build on freebsd"; false; fi
mkdir -p deps/freebsd_amd64

cd duckdb && \
CFLAGS="-O3" CXXFLAGS="-O3" ${DUCKDB_COMMON_BUILD_FLAGS} gmake bundle-library -j 2
Expand Down
104 changes: 49 additions & 55 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,7 @@ go get github.com/marcboeker/go-duckdb

```go
db, err := sql.Open("duckdb", "")
if err != nil {
...
}
check(err)
defer db.Close()
```

Expand All @@ -29,43 +27,38 @@ the file does not exist, then DuckDB creates it.

```go
db, err := sql.Open("duckdb", "/path/to/foo.db")
if err != nil {
...
}
check(err)
defer db.Close()
```

If you want to set specific [config options for DuckDB](https://duckdb.org/docs/sql/configuration), you can add them as query style parameters in the form of `name=value` pairs to the DSN.

```go
db, err := sql.Open("duckdb", "/path/to/foo.db?access_mode=read_only&threads=4")
if err != nil {
...
}
check(err)
defer db.Close()
```

Alternatively, you can use [sql.OpenDB](https://cs.opensource.google/go/go/+/refs/tags/go1.23.0:src/database/sql/sql.go;l=824). That way, you can perform initialization steps in a callback function before opening the database.
Here's an example that installs and loads the JSON extension when opening a database with `sql.OpenDB(connector)`.
Alternatively, you can use [sql.OpenDB](https://cs.opensource.google/go/go/+/refs/tags/go1.23.0:src/database/sql/sql.go;l=824).
That way, you can perform initialization steps in a callback function before opening the database.
Here's an example that configures some database parameters when opening a database with `sql.OpenDB(connector)`.

```go
connector, err := duckdb.NewConnector("/path/to/foo.db?access_mode=read_only&threads=4", func(execer driver.ExecerContext) error {
bootQueries := []string{
"INSTALL 'json'",
"LOAD 'json'",
"SET schema=main",
"SET search_path=main",
}

for _, query := range bootQueries {
_, err = execer.ExecContext(context.Background(), query, nil)
if err != nil {
...
return err
}
}
return nil
})
if err != nil {
...
}
check(err)

db := sql.OpenDB(connector)
defer db.Close()
Expand All @@ -87,46 +80,39 @@ conn, err := db.Conn(context.Background())
defer conn.Close()

rows, err := conn.QueryContext(context.Background(), "SELECT 42")
// alternatively, rows.Next() has to return false
// Alternatively, rows.Next() has to return false.
rows.Close()

appender, err := NewAppenderFromConn(conn, "", "test")
defer appender.Close()

// if not passed to sql.OpenDB
// If not passed to sql.OpenDB.
connector, err := NewConnector("", nil)
defer connector.Close()
```

## DuckDB Appender API

If you want to use the [DuckDB Appender API](https://duckdb.org/docs/data/appender.html), you can obtain a new `Appender` by passing a DuckDB connection to `NewAppenderFromConn()`. See `examples/appender.go` for a complete example.
If you want to use the [DuckDB Appender API](https://duckdb.org/docs/data/appender.html), you can obtain a new `Appender` by passing a DuckDB connection to `NewAppenderFromConn()`.
See `examples/appender.go` for a complete example.

```go
connector, err := duckdb.NewConnector("test.db", nil)
if err != nil {
...
}
check(err)
defer connector.Close()

conn, err := connector.Connect(context.Background())
if err != nil {
...
}
check(err)
defer conn.Close()

// obtain an appender from the connection
// NOTE: the table 'test_tbl' must exist in test.db
// Obtain an appender from the connection.
// NOTE: The table 'test_tbl' must exist in test.db.
appender, err := NewAppenderFromConn(conn, "", "test_tbl")
if err != nil {
...
}
check(err)
defer appender.Close()

err = appender.AppendRow(...)
if err != nil {
...
}
check(err)
```

## DuckDB Apache Arrow Interface
Expand All @@ -135,35 +121,29 @@ If you want to use the [DuckDB Arrow Interface](https://duckdb.org/docs/api/c/ap

```go
connector, err := duckdb.NewConnector("", nil)
if err != nil {
...
}
check(err)
defer connector.Close()

conn, err := connector.Connect(context.Background())
if err != nil {
...
}
check(err)
defer conn.Close()

// obtain the Arrow from the connection
// Obtain the Arrow from the connection.
arrow, err := duckdb.NewArrowFromConn(conn)
if err != nil {
...
}
check(err)

rdr, err := arrow.QueryContext(context.Background(), "SELECT * FROM generate_series(1, 10)")
if err != nil {
...
}
check(err)
defer rdr.Release()

for rdr.Next() {
// process records
// Process each record.
}
```

The Arrow interface is a heavy dependency. If you do not need it, you can disable it by passing `-tags=no_duckdb_arrow` to `go build`. This will be made opt-in in V2.
The Arrow interface is a heavy dependency.
If you do not need it, you can disable it by passing `-tags=no_duckdb_arrow` to `go build`.
This will be made opt-in in V2.

```sh
go build -tags="no_duckdb_arrow"
Expand All @@ -182,26 +162,40 @@ Now you can build your module as usual.

## Linking DuckDB

By default, `go-duckdb` statically links DuckDB into your binary. Statically linking DuckDB adds around 30 MB to your binary size. On Linux (Intel) and macOS (Intel and ARM), `go-duckdb` bundles pre-compiled static libraries for fast builds.
By default, `go-duckdb` statically links DuckDB into your binary.
Statically linking DuckDB increases your binary size.
`go-duckdb` bundles pre-compiled static libraries for some OS and architecture combinations.
- MacOS: amd64, arm64.
- Linux: amd64, arm64.
- FreeBSD: amd64.

Alternatively, you can dynamically link DuckDB by passing `-tags=duckdb_use_lib` to `go build`. You must have a copy of `libduckdb` available on your system (`.so` on Linux or `.dylib` on macOS), which you can download from the DuckDB [releases page](https://github.com/duckdb/duckdb/releases). For example:
Alternatively, you can dynamically link DuckDB by passing `-tags=duckdb_use_lib` to `go build`.
You must have a copy of `libduckdb` available on your system (`.so` on Linux or `.dylib` on macOS), which you can download from the DuckDB [releases page](https://github.com/duckdb/duckdb/releases).
For example:

```sh
# On Linux
# On Linux.
CGO_ENABLED=1 CGO_LDFLAGS="-L/path/to/libs" go build -tags=duckdb_use_lib main.go
LD_LIBRARY_PATH=/path/to/libs ./main

# On macOS
# On macOS.
CGO_ENABLED=1 CGO_LDFLAGS="-L/path/to/libs" go build -tags=duckdb_use_lib main.go
DYLD_LIBRARY_PATH=/path/to/libs ./main
```

## DuckDB Extensions

`go-duckdb` statically builds the `JSON` extension for its pre-compiled libraries.
Additionally, automatic extension loading is enabled.
The extensions available differ between the pre-compiled libraries.
Thus, if you fail to install and load an extension, you might have to link a custom DuckDB.

## Notes

`TIMESTAMP vs. TIMESTAMP_TZ`
**`TIMESTAMP vs. TIMESTAMP_TZ`**

In the C API, DuckDB stores both `TIMESTAMP` and `TIMESTAMP_TZ` as `duckdb_timestamp`, which holds the number of
microseconds elapsed since January 1, 1970 UTC (i.e., an instant without offset information).
When passing a `time.Time` to go-duckdb, go-duckdb transforms it to an instant with `UnixMicro()`,
even when using `TIMESTAMP_TZ`. Later, scanning either type of value returns an instant, as SQL types do not model
time zone information for individual values.
time zone information for individual values.
48 changes: 3 additions & 45 deletions duckdb_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,40 +51,18 @@ func TestOpen(t *testing.T) {
}

func TestConnectorBootQueries(t *testing.T) {
t.Run("many boot queries", func(t *testing.T) {
connector, err := NewConnector("", func(execer driver.ExecerContext) error {
bootQueries := []string{
"INSTALL 'json'",
"LOAD 'json'",
"SET schema=main",
"SET search_path=main",
}

for _, query := range bootQueries {
_, err := execer.ExecContext(context.Background(), query, nil)
require.NoError(t, err)
}
return nil
})
require.NoError(t, err)

db := sql.OpenDB(connector)
defer db.Close()
})

t.Run("readme example", func(t *testing.T) {
db, err := sql.Open("duckdb", "foo.db")
require.NoError(t, err)
_ = db.Close()

connector, err := NewConnector("foo.db?access_mode=read_only&threads=4", func(execer driver.ExecerContext) error {
bootQueries := []string{
"INSTALL 'json'",
"LOAD 'json'",
"SET schema=main",
"SET search_path=main",
}

for _, query := range bootQueries {
_, err := execer.ExecContext(context.Background(), query, nil)
_, err = execer.ExecContext(context.Background(), query, nil)
require.NoError(t, err)
}
return nil
Expand Down Expand Up @@ -148,17 +126,6 @@ func TestConnPool(t *testing.T) {

func TestConnInit(t *testing.T) {
connector, err := NewConnector("", func(execer driver.ExecerContext) error {
bootQueries := []string{
"INSTALL 'json'",
"LOAD 'json'",
}

for _, qry := range bootQueries {
_, err := execer.ExecContext(context.Background(), qry, nil)
if err != nil {
return err
}
}
return nil
})
require.NoError(t, err)
Expand Down Expand Up @@ -280,8 +247,6 @@ func TestQuery(t *testing.T) {
func TestJSON(t *testing.T) {
t.Parallel()
db := openDB(t)

loadJSONExt(t, db)
var data string

t.Run("select empty JSON", func(t *testing.T) {
Expand Down Expand Up @@ -671,13 +636,6 @@ func openDB(t *testing.T) *sql.DB {
return db
}

func loadJSONExt(t *testing.T, db *sql.DB) {
_, err := db.Exec("INSTALL 'json'")
require.NoError(t, err)
_, err = db.Exec("LOAD 'json'")
require.NoError(t, err)
}

func createTable(db *sql.DB, t *testing.T, sql string) *sql.Result {
res, err := db.Exec(sql)
require.NoError(t, err)
Expand Down

0 comments on commit 31fc9de

Please sign in to comment.