Merge ssh://github.com/davidhcoe/arrow-adbc into dev/go-flight-sql-in…

…terop

.env

@@ -33,7 +33,7 @@ MANYLINUX=2-28
 MAVEN=3.6.3
 PLATFORM=linux/amd64
 PYTHON=3.9
-GO=1.22.4
+GO=1.22.9
 ARROW_MAJOR_VERSION=14
 DOTNET=8.0
 

.github/workflows/packaging.yml

@@ -132,8 +132,9 @@ jobs:
           popd
 
       - name: Upload Go binaries
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          name: go-${{ matrix.os }}
           retention-days: 7
           path: |
             adbc/go/adbc/pkg/libadbc_driver_flightsql.*
@@ -167,13 +168,15 @@ jobs:
           echo "schedule: ${{ github.event.schedule }}" >> $GITHUB_STEP_SUMMARY
           echo "ref: ${{ github.ref }}" >> $GITHUB_STEP_SUMMARY
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
+          pattern: go-*
           path: adbc/go/adbc/pkg
+          merge-multiple: true
 
       - name: Copy Go binaries
         run: |
-          pushd adbc/go/adbc/pkg/artifact
+          pushd adbc/go/adbc/pkg/
           cp *.dll ../
           cp *.so ../
           cp *.dylib ../

CHANGELOG.md

@@ -699,3 +699,63 @@
 - **ci**: update website_build.sh for new versioning scheme (#1972)
 - **dev/release**: update C# <VersionSuffix> tag (#1973)
 - **c/vendor/nanoarrow**: Fix -Wreorder warning (#1966)
+
+## ADBC Libraries 15 (2024-11-08)
+
+### Versions
+
+- C/C++/GLib/Go/Python/Ruby: 1.3.0
+- C#: 0.15.0
+- Java: 0.15.0
+- R: 0.15.0
+- Rust: 0.15.0
+
+### Feat
+
+- **c/driver/postgresql**: Enable basic connect/query workflow for Redshift (#2219)
+- **rust/drivers/datafusion**: add support for bulk ingest (#2279)
+- **csharp/src/Drivers/Apache**: convert Double to Float for Apache Spark on scalar conversion (#2296)
+- **go/adbc/driver/snowflake**: update to the latest 1.12.0 gosnowflake driver (#2298)
+- **csharp/src/Drivers/BigQuery**: support max stream count setting when creating read session (#2289)
+- **rust/drivers**: adbc driver for datafusion (#2267)
+- **go/adbc/driver/snowflake**: improve GetObjects performance and semantics (#2254)
+- **c**: Implement ingestion and testing for float16, string_view, and binary_view (#2234)
+- **r**: Add R BigQuery driver wrapper (#2235)
+- **csharp/src/Drivers/Apache/Spark**: add request_timeout_ms option to allow longer HTTP request length (#2218)
+- **go/adbc/driver/snowflake**: add support for a client config file (#2197)
+- **csharp/src/Client**: Additional parameter support for DbCommand (#2195)
+- **csharp/src/Drivers/Apache/Spark**: add option to ignore TLS/SSL certificate exceptions (#2188)
+- **csharp/src/Drivers/Apache/Spark**: Perform scalar data type conversion for Spark over HTTP (#2152)
+- **csharp/src/Drivers/Apache/Spark**: Azure HDInsight Spark Documentation (#2164)
+- **c/driver/postgresql**: Implement ingestion of list types for PostgreSQL (#2153)
+- **csharp/src/Drivers/Apache/Spark**: poc - Support for Apache Spark over HTTP (non-Arrow) (#2018)
+- **c/driver/postgresql**: add `arrow.opaque` type metadata (#2122)
+
+### Fix
+
+- **csharp/src/Drivers/Apache**: fix float data type handling for tests on Databricks Spark (#2283)
+- **go/adbc/driver/internal/driverbase**: proper unmarshalling for ConstraintColumnNames (#2285)
+- **csharp/src/Drivers/Apache**: fix to workaround concurrency issue (#2282)
+- **csharp/src/Drivers/Apache**: correctly handle empty response and add Client tests (#2275)
+- **csharp/src/Drivers/Apache**: remove interleaved async look-ahead code (#2273)
+- **c/driver_manager**: More robust error reporting for errors that occur before AdbcDatabaseInit() (#2266)
+- **rust**: implement database/connection constructors without options (#2242)
+- **csharp/src/Drivers**: update System.Text.Json to version 8.0.5 because of known vulnerability (#2238)
+- **csharp/src/Drivers/Apache/Spark**: correct batch handling for the HiveServer2Reader (#2215)
+- **go/adbc/driver/snowflake**: call GetObjects with null catalog at catalog depth (#2194)
+- **csharp/src/Drivers/Apache/Spark**: correct BatchSize implementation for base reader (#2199)
+- **csharp/src/Drivers/Apache/Spark**: correct precision/scale handling with zeros in fractional portion (#2198)
+- **csharp/src/Drivers/BigQuery**: Fixed GBQ driver issue when results.TableReference is null (#2165)
+- **go/adbc/driver/snowflake**: fix setting database and schema context after initial connection (#2169)
+- **csharp/src/Drivers/Interop/Snowflake**: add test to demonstrate DEFAULT_ROLE behavior (#2151)
+- **c/driver/postgresql**: Improve error reporting for queries that error before the COPY header is sent (#2134)
+
+### Refactor
+
+- **c/driver/postgresql**: cleanups for result_helper signatures (#2261)
+- **c/driver/postgresql**: Use GetObjectsHelper from framework to build objects (#2189)
+- **csharp/src/Drivers/Apache/Spark**: use UTF8 string for data conversion, instead of .NET String (#2192)
+- **c/driver/postgresql**: Use Status for error handling in BindStream (#2187)
+- **c/driver/postgresql**: Use Status instead of AdbcStatusCode/AdbcError in result helper (#2178)
+- **c/driver**: Use non-objects framework components in Postgres driver (#2166)
+- **c/driver/postgresql**: Use copy writer in BindStream for parameter binding (#2157)

c/cmake_modules/AdbcDefines.cmake

@@ -93,7 +93,9 @@ if(MSVC)
   # Don't warn about padding added after members
   add_compile_options(/wd4820)
   add_compile_options(/wd5027)
+  add_compile_options(/wd5039)
   add_compile_options(/wd5045)
+  add_compile_options(/wd5246)
 elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang"
        OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang"
        OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU")

c/cmake_modules/AdbcVersion.cmake

@@ -21,7 +21,7 @@
 # ------------------------------------------------------------
 # Version definitions
 
-set(ADBC_VERSION "1.3.0-SNAPSHOT")
+set(ADBC_VERSION "1.4.0-SNAPSHOT")
 string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ADBC_BASE_VERSION "${ADBC_VERSION}")
 string(REPLACE "." ";" _adbc_version_list "${ADBC_BASE_VERSION}")
 list(GET _adbc_version_list 0 ADBC_VERSION_MAJOR)

c/driver/framework/base_driver.h

@@ -116,8 +116,9 @@ class Option {
                                              "': trailing data", value);
             }
             return parsed;
+          } else {
+            return status::InvalidArgument("Invalid integer value ", this->Format());
           }
-          return status::InvalidArgument("Invalid integer value ", this->Format());
         },
         value_);
   }
@@ -129,8 +130,9 @@ class Option {
           using T = std::decay_t<decltype(value)>;
           if constexpr (std::is_same_v<T, std::string>) {
             return value;
+          } else {
+            return status::InvalidArgument("Invalid string value ", this->Format());
           }
-          return status::InvalidArgument("Invalid string value ", this->Format());
         },
         value_);
   }

c/driver/framework/utility.cc

@@ -163,7 +163,6 @@ Status MakeGetInfoStream(const std::vector<InfoValue>& infos, ArrowArrayStream*
           } else {
             static_assert(!sizeof(T), "info value type not implemented");
           }
-          return status::Ok();
         },
         info.value));
     UNWRAP_ERRNO(Internal, ArrowArrayFinishElement(array.get()));

c/driver/postgresql/connection.cc

@@ -300,7 +300,8 @@ class PostgresGetObjectsHelper : public adbc::driver::GetObjectsHelper {
 
     Column col;
     col.column_name = next_column_[0].value();
-    UNWRAP_RESULT(col.ordinal_position, next_column_[1].ParseInteger());
+    UNWRAP_RESULT(int64_t ordinal_position, next_column_[1].ParseInteger());
+    col.ordinal_position = static_cast<int32_t>(ordinal_position);
     if (!next_column_[2].is_null) {
       col.remarks = next_column_[2].value();
     }

c/driver/postgresql/database.cc

@@ -294,7 +294,7 @@ static Status InsertPgAttributeResult(
     UNWRAP_RESULT(int64_t col_oid, item[2].ParseInteger());
 
     if (type_oid != current_type_oid && !columns.empty()) {
-      resolver->InsertClass(current_type_oid, columns);
+      resolver->InsertClass(static_cast<uint32_t>(current_type_oid), columns);
       columns.clear();
       current_type_oid = type_oid;
     }
@@ -347,12 +347,12 @@ static Status InsertPgTypeResult(const PqResultHelper& result,
     type_item.class_oid = static_cast<uint32_t>(typrelid);
     type_item.base_oid = static_cast<uint32_t>(typbasetype);
 
-    int result = resolver->Insert(type_item, nullptr);
+    int insert_result = resolver->Insert(type_item, nullptr);
 
     // If there's an array type and the insert succeeded, add that now too
-    if (result == NANOARROW_OK && typarray != 0) {
+    if (insert_result == NANOARROW_OK && typarray != 0) {
       std::string array_typname = "_" + std::string(typname);
-      type_item.oid = typarray;
+      type_item.oid = static_cast<uint32_t>(typarray);
       type_item.typname = array_typname.c_str();
       type_item.typreceive = "array_recv";
       type_item.child_oid = static_cast<uint32_t>(oid);

c/driver/postgresql/result_helper.cc

@@ -46,7 +46,7 @@ Status PqResultHelper::PrepareInternal(int n_params, const Oid* param_oids) cons
 Status PqResultHelper::Prepare() const { return PrepareInternal(0, nullptr); }
 
 Status PqResultHelper::Prepare(const std::vector<Oid>& param_oids) const {
-  return PrepareInternal(param_oids.size(), param_oids.data());
+  return PrepareInternal(static_cast<int>(param_oids.size()), param_oids.data());
 }
 
 Status PqResultHelper::DescribePrepared() {
@@ -90,8 +90,8 @@ Status PqResultHelper::Execute(const std::vector<std::string>& params,
     }
 
     ClearResult();
-    result_ = PQexecParams(conn_, query_.c_str(), param_values.size(), param_oids_ptr,
-                           param_values.data(), param_lengths.data(),
+    result_ = PQexecParams(conn_, query_.c_str(), static_cast<int>(param_values.size()),
+                           param_oids_ptr, param_values.data(), param_lengths.data(),
                            param_formats.data(), static_cast<int>(output_format_));
   }
 

c/driver/postgresql/result_reader.cc

@@ -100,8 +100,8 @@ int PqResultArrayReader::GetNext(struct ArrowArray* out) {
         item.size_bytes = pg_item.len;
       }
 
-      NANOARROW_RETURN_NOT_OK(
-          field_readers_[i]->Read(&item, item.size_bytes, tmp->children[i], &na_error_));
+      NANOARROW_RETURN_NOT_OK(field_readers_[i]->Read(
+          &item, static_cast<int32_t>(item.size_bytes), tmp->children[i], &na_error_));
     }
   }
 

c/driver/postgresql/statement.cc

@@ -546,22 +546,23 @@ AdbcStatusCode PostgresStatement::ExecuteSchema(struct ArrowSchema* schema,
   PqResultHelper helper(connection_->conn(), query_);
 
   if (bind_.release) {
-    nanoarrow::UniqueSchema schema;
+    nanoarrow::UniqueSchema param_schema;
     struct ArrowError na_error;
     ArrowErrorInit(&na_error);
-    CHECK_NA_DETAIL(INTERNAL, ArrowArrayStreamGetSchema(&bind_, schema.get(), &na_error),
+    CHECK_NA_DETAIL(INTERNAL,
+                    ArrowArrayStreamGetSchema(&bind_, param_schema.get(), &na_error),
                     &na_error, error);
 
-    if (std::string(schema->format) != "+s") {
+    if (std::string(param_schema->format) != "+s") {
       SetError(error, "%s", "[libpq] Bind parameters must have type STRUCT");
       return ADBC_STATUS_INVALID_STATE;
     }
 
-    std::vector<Oid> param_oids(schema->n_children);
-    for (int64_t i = 0; i < schema->n_children; i++) {
+    std::vector<Oid> param_oids(param_schema->n_children);
+    for (int64_t i = 0; i < param_schema->n_children; i++) {
       PostgresType pg_type;
       CHECK_NA_DETAIL(INTERNAL,
-                      PostgresType::FromSchema(*type_resolver_, schema->children[i],
+                      PostgresType::FromSchema(*type_resolver_, param_schema->children[i],
                                                &pg_type, &na_error),
                       &na_error, error);
       param_oids[i] = pg_type.oid();

c/driver/sqlite/sqlite.cc

@@ -150,15 +150,15 @@ class SqliteQuery {
     return Close(rc);
   }
 
-  Status Close(int rc) {
+  Status Close(int last_rc) {
     if (stmt_) {
       int rc = sqlite3_finalize(stmt_);
       stmt_ = nullptr;
       if (rc != SQLITE_OK && rc != SQLITE_DONE) {
         return status::fmt::Internal("failed to execute: {}\nquery was: {}",
                                      sqlite3_errmsg(conn_), query_);
       }
-    } else if (rc != SQLITE_OK) {
+    } else if (last_rc != SQLITE_OK) {
       return status::fmt::Internal("failed to execute: {}\nquery was: {}",
                                    sqlite3_errmsg(conn_), query_);
     }
@@ -192,7 +192,7 @@ class SqliteQuery {
       UNWRAP_RESULT(bool has_row, q.Next());
       if (!has_row) break;
 
-      int rc = std::forward<RowFunc>(row_func)(q.stmt_);
+      rc = std::forward<RowFunc>(row_func)(q.stmt_);
       if (rc != SQLITE_OK) break;
     }
     return q.Close();

c/driver/sqlite/statement_reader.c

@@ -334,8 +334,8 @@ AdbcStatusCode AdbcSqliteBinderBindNext(struct AdbcSqliteBinder* binder, sqlite3
         case NANOARROW_TYPE_BINARY_VIEW: {
           struct ArrowBufferView value =
               ArrowArrayViewGetBytesUnsafe(binder->batch.children[col], binder->next_row);
-          status = sqlite3_bind_blob(stmt, col + 1, value.data.as_char, value.size_bytes,
-                                     SQLITE_STATIC);
+          status = sqlite3_bind_blob(stmt, col + 1, value.data.as_char,
+                                     (int)value.size_bytes, SQLITE_STATIC);
           break;
         }
         case NANOARROW_TYPE_BOOL:
@@ -377,8 +377,8 @@ AdbcStatusCode AdbcSqliteBinderBindNext(struct AdbcSqliteBinder* binder, sqlite3
         case NANOARROW_TYPE_STRING_VIEW: {
           struct ArrowBufferView value =
               ArrowArrayViewGetBytesUnsafe(binder->batch.children[col], binder->next_row);
-          status = sqlite3_bind_text(stmt, col + 1, value.data.as_char, value.size_bytes,
-                                     SQLITE_STATIC);
+          status = sqlite3_bind_text(stmt, col + 1, value.data.as_char,
+                                     (int)value.size_bytes, SQLITE_STATIC);
           break;
         }
         case NANOARROW_TYPE_DICTIONARY: {
@@ -391,7 +391,7 @@ AdbcStatusCode AdbcSqliteBinderBindNext(struct AdbcSqliteBinder* binder, sqlite3
             struct ArrowBufferView value = ArrowArrayViewGetBytesUnsafe(
                 binder->batch.children[col]->dictionary, value_index);
             status = sqlite3_bind_text(stmt, col + 1, value.data.as_char,
-                                       value.size_bytes, SQLITE_STATIC);
+                                       (int)value.size_bytes, SQLITE_STATIC);
           }
           break;
         }
@@ -411,16 +411,16 @@ AdbcStatusCode AdbcSqliteBinderBindNext(struct AdbcSqliteBinder* binder, sqlite3
 
           RAISE_ADBC(ArrowDate32ToIsoString((int32_t)value, &tsstr, error));
           // SQLITE_TRANSIENT ensures the value is copied during bind
-          status =
-              sqlite3_bind_text(stmt, col + 1, tsstr, strlen(tsstr), SQLITE_TRANSIENT);
+          status = sqlite3_bind_text(stmt, col + 1, tsstr, (int)strlen(tsstr),
+                                     SQLITE_TRANSIENT);
 
           free(tsstr);
           break;
         }
         case NANOARROW_TYPE_TIMESTAMP: {
           struct ArrowSchemaView bind_schema_view;
-          RAISE_ADBC(ArrowSchemaViewInit(&bind_schema_view, binder->schema.children[col],
-                                         &arrow_error));
+          RAISE_NA(ArrowSchemaViewInit(&bind_schema_view, binder->schema.children[col],
+                                       &arrow_error));
           enum ArrowTimeUnit unit = bind_schema_view.time_unit;
           int64_t value =
               ArrowArrayViewGetIntUnsafe(binder->batch.children[col], binder->next_row);
@@ -429,8 +429,8 @@ AdbcStatusCode AdbcSqliteBinderBindNext(struct AdbcSqliteBinder* binder, sqlite3
           RAISE_ADBC(ArrowTimestampToIsoString(value, unit, &tsstr, error));
 
           // SQLITE_TRANSIENT ensures the value is copied during bind
-          status =
-              sqlite3_bind_text(stmt, col + 1, tsstr, strlen(tsstr), SQLITE_TRANSIENT);
+          status = sqlite3_bind_text(stmt, col + 1, tsstr, (int)strlen(tsstr),
+                                     SQLITE_TRANSIENT);
           free((char*)tsstr);
           break;
         }
@@ -844,7 +844,7 @@ AdbcStatusCode StatementReaderUpcastInt64ToDouble(struct ArrowBuffer* data,
   size_t num_elements = data->size_bytes / sizeof(int64_t);
   const int64_t* elements = (const int64_t*)data->data;
   for (size_t i = 0; i < num_elements; i++) {
-    double value = elements[i];
+    double value = (double)elements[i];
     ArrowBufferAppendUnsafe(&doubles, &value, sizeof(double));
   }
   ArrowBufferReset(data);
@@ -1133,7 +1133,7 @@ AdbcStatusCode AdbcSqliteExportReader(sqlite3* db, sqlite3_stmt* stmt,
   memset(reader, 0, sizeof(struct StatementReader));
   reader->db = db;
   reader->stmt = stmt;
-  reader->batch_size = batch_size;
+  reader->batch_size = (int)batch_size;
 
   stream->private_data = reader;
   stream->release = StatementReaderRelease;

c/meson.build

@@ -18,7 +18,7 @@
 project(
     'arrow-adbc',
     'c', 'cpp',
-    version: '1.3.0-SNAPSHOT',
+    version: '1.4.0-SNAPSHOT',
     license: 'Apache-2.0',
     meson_version: '>=1.3.0',
     default_options: [

ci/conda/meta.yaml

@@ -17,8 +17,7 @@
 
 package:
   name: arrow-adbc-split
-  # TODO: this needs to get bumped by the release process
-  version: 1.3.0
+  version: 1.4.0
 
 source:
   path: ../../