build(python): add CMake build target for Python (apache#1257)

davidhcoe · Dec 6, 2023 · cf3b5c0 · cf3b5c0
1 parent 414379d
commit cf3b5c0
Show file tree

Hide file tree

Showing 9 changed files with 123 additions and 114 deletions.
diff --git a/c/CMakeLists.txt b/c/CMakeLists.txt
@@ -67,5 +67,53 @@ if(ADBC_INTEGRATION_DUCKDB)
   add_subdirectory(integration/duckdb)
 endif()
 
+if(ADBC_BUILD_PYTHON)
+  find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
+
+  if(NOT ADBC_BUILD_SHARED)
+    message(FATAL_ERROR "Building Python requires ADBC_BUILD_SHARED=ON")
+  endif()
+
+  # NB: the Python packages require the driver manager to be installed,
+  # but you don't technically need -DADBC_DRIVER_MANAGER=ON when installing
+  # other Python packages. To be safe then, we always install the driver
+  # manager package, regardless of the value of -DABC_DRIVER_MANAGER
+  # --config-settings eidtable_mode=compat required due to
+  # https://github.com/python/mypy/issues/13392
+  add_custom_target(python
+                    COMMAND ${Python3_EXECUTABLE} -m pip install --no-deps -e
+                            "${REPOSITORY_ROOT}/python/adbc_driver_manager"
+                            --config-settings editable_mode=compat)
+
+  macro(adbc_install_python_package TARGET)
+    string(TOUPPER ${TARGET} ${TARGET}_LIB_upper)
+    add_custom_command(TARGET python
+                       POST_BUILD
+                       COMMAND ${CMAKE_COMMAND} -E env "ADBC_${${TARGET}_LIB_upper}_\
+LIBRARY=$<TARGET_FILE:adbc_driver_${TARGET}_shared>" ${Python3_EXECUTABLE} -m pip install
+                               --no-deps -e
+                               "${REPOSITORY_ROOT}/python/adbc_driver_${TARGET}"
+                       COMMENT "pip installing the adbc_driver_${TARGET} library..."
+                               DEPENDS $<TARGET_FILE:adbc_driver_${TARGET}_shared>
+                       WORKING_DIRECTORY ${REPOSITORY_ROOT})
+  endmacro()
+
+  if(ADBC_DRIVER_POSTGRESQL)
+    adbc_install_python_package(postgresql)
+  endif()
+
+  if(ADBC_DRIVER_SQLITE)
+    adbc_install_python_package(sqlite)
+  endif()
+
+  if(ADBC_DRIVER_FLIGHTSQL)
+    adbc_install_python_package(flightsql)
+  endif()
+
+  if(ADBC_DRIVER_SNOWFLAKE)
+    adbc_install_python_package(snowflake)
+  endif()
+endif()
+
 validate_config()
 config_summary_message()
diff --git a/c/cmake_modules/GoUtils.cmake b/c/cmake_modules/GoUtils.cmake
@@ -153,7 +153,7 @@ function(add_go_lib GO_MOD_DIR GO_LIBNAME)
     add_custom_target(${GO_LIBNAME}_target ALL
                       DEPENDS "${LIBOUT_SHARED}.${ADBC_FULL_SO_VERSION}"
                               "${LIBOUT_SHARED}.${ADBC_SO_VERSION}" "${LIBOUT_SHARED}")
-    add_library(${GO_LIBNAME}_shared SHARED IMPORTED)
+    add_library(${GO_LIBNAME}_shared SHARED IMPORTED GLOBAL)
     set_target_properties(${GO_LIBNAME}_shared
                           PROPERTIES IMPORTED_LOCATION
                                      "${LIBOUT_SHARED}.${ADBC_FULL_SO_VERSION}"

diff --git a/ci/scripts/python_build.ps1 b/ci/scripts/python_build.ps1
@@ -20,7 +20,6 @@ $ErrorActionPreference = "Stop"
 
 $SourceDir = $Args[0]
 $BuildDir = $Args[1]
-$InstallDir = if ($Args[2] -ne $null) { $Args[2] } else { Join-Path $BuildDir "local/" }
 
 $BuildAll = $env:BUILD_ALL -ne "0"
 $BuildDriverFlightSql = ($BuildAll -and (-not ($env:BUILD_DRIVER_FLIGHTSQL -eq "0"))) -or ($env:BUILD_DRIVER_FLIGHTSQL -eq "1")
@@ -29,70 +28,11 @@ $BuildDriverPostgreSQL = ($BuildAll -and (-not ($env:BUILD_DRIVER_POSTGRESQL -eq
 $BuildDriverSqlite = ($BuildAll -and (-not ($env:BUILD_DRIVER_SQLITE -eq "0"))) -or ($env:BUILD_DRIVER_SQLITE -eq "1")
 $BuildDriverSnowflake = ($BuildAll -and (-not ($env:BUILD_DRIVER_SNOWFLAKE -eq "0"))) -or ($env:BUILD_DRIVER_SNOWFLAKE -eq "1")
 
-function Build-Subproject {
-    $Subproject = $Args[0]
-    $SubprojectBuild = Join-Path $SourceDir "python\$($Subproject)"
-
-    echo "============================================================"
-    echo "Building $($Subproject)"
-    echo "============================================================"
-
-    pip install -e $SubprojectBuild
-    if (-not $?) { exit 1 }
-}
-
-if ($BuildDriverManager) {
-    Build-Subproject adbc_driver_manager
-}
-if ($BuildDriverFlightSql) {
-    $env:ADBC_FLIGHTSQL_LIBRARY = Get-Childitem `
-      -ErrorAction SilentlyContinue `
-      -Path $InstallDir `
-      -Recurse `
-      -Include "adbc_driver_flightsql.dll","libadbc_driver_flightsql.so" | % {$_.FullName}
-    echo $env:ADBC_FLIGHTSQL_LIBRARY
-    if ($env:ADBC_FLIGHTSQL_LIBRARY -eq $null) {
-        echo "Could not find Flight SQL driver in $($InstallDir)"
-        exit 1
-    }
-    Build-Subproject adbc_driver_flightsql
-}
-if ($BuildDriverPostgreSQL) {
-    $env:ADBC_POSTGRESQL_LIBRARY = Get-Childitem `
-      -ErrorAction SilentlyContinue `
-      -Path $InstallDir `
-      -Recurse `
-      -Include "adbc_driver_postgresql.dll","libadbc_driver_postgresql.so" | % {$_.FullName}
-    echo $env:ADBC_POSTGRESQL_LIBRARY
-    if ($env:ADBC_POSTGRESQL_LIBRARY -eq $null) {
-        echo "Could not find libpq driver in $($InstallDir)"
-        exit 1
-    }
-    Build-Subproject adbc_driver_postgresql
-}
-if ($BuildDriverSqlite) {
-    $env:ADBC_SQLITE_LIBRARY = Get-Childitem `
-      -ErrorAction SilentlyContinue `
-      -Path $InstallDir `
-      -Recurse `
-      -Include "adbc_driver_sqlite.dll","libadbc_driver_sqlite.so" | % {$_.FullName}
-    echo $env:ADBC_SQLITE_LIBRARY
-    if ($env:ADBC_SQLITE_LIBRARY -eq $null) {
-        echo "Could not find SQLite driver in $($InstallDir)"
-        exit 1
-    }
-    Build-Subproject adbc_driver_sqlite
-}
-if ($BuildDriverSnowflake) {
-    $env:ADBC_SNOWFLAKE_LIBRARY = Get-Childitem `
-      -ErrorAction SilentlyContinue `
-      -Path $InstallDir `
-      -Recurse `
-      -Include "adbc_driver_snowflake.dll","libadbc_driver_snowflake.so" | % {$_.FullName}
-    echo $env:ADBC_SNOWFLAKE_LIBRARY
-    if ($env:ADBC_SNOWFLAKE_LIBRARY -eq $null) {
-        echo "Could not find Snowflake driver in $($InstallDir)"
-        exit 1
-    }
-    Build-Subproject adbc_driver_snowflake
-}
+cmake -S "$($SourceDir)\c" -B $BuildDir `
+    -DADBC_DRIVER_MANAGER=$BuildDriverManager `
+    -DADBC_DRIVER_FLIGHTSQL=$BuildDriverFlightSql `
+    -DADBC_DRIVER_POSTGRESQL=$BuildDriverPostgreSQL `
+    -DADBC_DRIVER_SQLITE=$BuildDriverSqlite `
+    -DADBC_DRIVER_SNOWFLAKE=$BuildDriverSnowflake `
+    -DADBC_BUILD_PYTHON=ON
+cmake --build $BuildDir --target python
diff --git a/ci/scripts/python_build.sh b/ci/scripts/python_build.sh
@@ -25,58 +25,23 @@ set -e
 : ${BUILD_DRIVER_SQLITE:=${BUILD_ALL}}
 : ${BUILD_DRIVER_SNOWFLAKE:=${BUILD_ALL}}
 
-if [[ $(uname) = "Darwin" ]]; then
-    ADBC_LIBRARY_SUFFIX="dylib"
-else
-    ADBC_LIBRARY_SUFFIX="so"
-fi
-
-build_subproject() {
-    local -r source_dir="${1}"
-    local -r install_dir="${2}"
-    local -r subproject="${3}"
-
-    if [[ "${subproject}" = "adbc_driver_flightsql" ]]; then
-        export ADBC_FLIGHTSQL_LIBRARY="${install_dir}/lib/libadbc_driver_flightsql.${ADBC_LIBRARY_SUFFIX}"
-    elif [[ "${subproject}" = "adbc_driver_postgresql" ]]; then
-        export ADBC_POSTGRESQL_LIBRARY="${install_dir}/lib/libadbc_driver_postgresql.${ADBC_LIBRARY_SUFFIX}"
-    elif [[ "${subproject}" = "adbc_driver_sqlite" ]]; then
-        export ADBC_SQLITE_LIBRARY="${install_dir}/lib/libadbc_driver_sqlite.${ADBC_LIBRARY_SUFFIX}"
-    elif [[ "${subproject}" = "adbc_driver_snowflake" ]]; then
-        export ADBC_SNOWFLAKE_LIBRARY="${install_dir}/lib/libadbc_driver_snowflake.${ADBC_LIBRARY_SUFFIX}"
-    fi
-
-    python -m pip install --no-deps "${source_dir}/python/${subproject}"
-}
 
 main() {
     local -r source_dir="${1}"
     local -r build_dir="${2}"
-    local install_dir="${3}"
-
-    if [[ -z "${install_dir}" ]]; then
-        install_dir="${build_dir}/local"
-    fi
-
-    if [[ "${BUILD_DRIVER_FLIGHTSQL}" -gt 0 ]]; then
-        build_subproject "${source_dir}" "${install_dir}" adbc_driver_flightsql
-    fi
-
-    if [[ "${BUILD_DRIVER_MANAGER}" -gt 0 ]]; then
-        build_subproject "${source_dir}" "${install_dir}" adbc_driver_manager
-    fi
 
-    if [[ "${BUILD_DRIVER_POSTGRESQL}" -gt 0 ]]; then
-        build_subproject "${source_dir}" "${install_dir}" adbc_driver_postgresql
-    fi
+    set -x
 
-    if [[ "${BUILD_DRIVER_SQLITE}" -gt 0 ]]; then
-        build_subproject "${source_dir}" "${install_dir}" adbc_driver_sqlite
-    fi
+    cmake -S "${source_dir}/c" -B ${build_dir} \
+          -DADBC_DRIVER_MANAGER=${BUILD_DRIVER_MANAGER} \
+          -DADBC_DRIVER_FLIGHTSQL=${BUILD_DRIVER_FLIGHTSQL} \
+          -DADBC_DRIVER_POSTGRESQL=${BUILD_DRIVER_POSTGRESQL} \
+          -DADBC_DRIVER_SQLITE=${BUILD_DRIVER_SQLITE} \
+          -DADBC_DRIVER_SNOWFLAKE=${BUILD_DRIVER_SNOWFLAKE} \
+          -DADBC_BUILD_PYTHON=ON
+    cmake --build ${build_dir} --target python
 
-    if [[ "${BUILD_DRIVER_SNOWFLAKE}" -gt 0 ]]; then
-        build_subproject "${source_dir}" "${install_dir}" adbc_driver_snowflake
-    fi
+    set +x
 }
 
 main "$@"
diff --git a/python/adbc_driver_flightsql/README.md b/python/adbc_driver_flightsql/README.md
@@ -44,6 +44,18 @@ export ADBC_FLIGHTSQL_LIBRARY=/path/to/libadbc_driver_flightsql.so
 pip install --no-deps -e .
 ```
 
+For users building from the arrow-adbc source repository, you can alternately use CMake to manage library dependencies and set environment variables for you. Assuming you specify ``-DADBC_DRIVER_FLIGHTSQL=ON`` you can also add ``-DADBC_BUILD_PYTHON=ON`` to define a ``python`` target.
+
+For example, assuming you run cmake from the project root:
+
+```shell
+cmake -S c -B build --preset debug -DADBC_BUILD_PYTHON=ON
+cmake --build build --target python
+```
+
+will properly build and install the Python library for you.
+
+
 See [CONTRIBUTING.md](../../CONTRIBUTING.md) for details on the
 general build process.
 

diff --git a/python/adbc_driver_manager/README.md b/python/adbc_driver_manager/README.md
@@ -38,6 +38,17 @@ row-oriented API of the base DBAPI interface.
 
 Dependencies: a C++ compiler.
 
+For users building from the arrow-adbc source repository, you can alternately use CMake to manage library dependencies and set environment variables for you. You can add ``-DADBC_BUILD_PYTHON=ON`` to define a ``python`` target.
+
+For example, assuming you run cmake from the project root:
+
+```shell
+cmake -S c -B build --preset debug -DADBC_BUILD_PYTHON=ON
+cmake --build build --target python
+```
+
+will properly build and install the Python library for you.
+
 See [CONTRIBUTING.md](../../CONTRIBUTING.md) for details.
 
 ## Testing

diff --git a/python/adbc_driver_postgresql/README.md b/python/adbc_driver_postgresql/README.md
@@ -57,6 +57,17 @@ export ADBC_POSTGRESQL_LIBRARY=/path/to/libadbc_driver_postgresql.so
 pip install --no-deps -e .
 ```
 
+For users building from the arrow-adbc source repository, you can alternately use CMake to manage library dependencies and set environment variables for you. Assuming you specify ``-DADBC_DRIVER_POSTGRESQL=ON`` you can also add ``-DADBC_BUILD_PYTHON=ON`` to define a ``python`` target.
+
+For example, assuming you run cmake from the project root:
+
+```shell
+cmake -S c -B build --preset debug -DADBC_BUILD_PYTHON=ON
+cmake --build build --target python
+```
+
+will properly build and install the Python library for you.
+
 See [CONTRIBUTING.md](../../CONTRIBUTING.md) for details on the
 general build process.
 

diff --git a/python/adbc_driver_snowflake/README.md b/python/adbc_driver_snowflake/README.md
@@ -44,6 +44,17 @@ export ADBC_SNOWFLAKE_LIBRARY=/path/to/libadbc_driver_snowflake.so
 pip install --no-deps -e .
 ```
 
+For users building from the arrow-adbc source repository, you can alternately use CMake to manage library dependencies and set environment variables for you. Assuming you specify ``-DADBC_DRIVER_SNOWFLAKE=ON`` you can also add ``-DADBC_BUILD_PYTHON=ON`` to define a ``python`` target.
+
+For example, assuming you run cmake from the project root:
+
+```shell
+cmake -S c -B build --preset debug -DADBC_BUILD_PYTHON=ON
+cmake --build build --target python
+```
+
+will properly build and install the Python library for you.
+
 See [CONTRIBUTING.md](../../CONTRIBUTING.md) for details on the
 general build process.
 

diff --git a/python/adbc_driver_sqlite/README.md b/python/adbc_driver_sqlite/README.md
@@ -55,6 +55,17 @@ export ADBC_SQLITE_LIBRARY=/path/to/libadbc_driver_sqlite.so
 pip install --no-deps -e .
 ```
 
+For users building from the arrow-adbc source repository, you can alternately use CMake to manage library dependencies and set environment variables for you. Assuming you specify ``-DADBC_DRIVER_SQLITE=ON`` you can also add ``-DADBC_BUILD_PYTHON=ON`` to define a ``python`` target.
+
+For example, assuming you run cmake from the project root:
+
+```shell
+cmake -S c -B build --preset debug -DADBC_BUILD_PYTHON=ON
+cmake --build build --target python
+```
+
+will properly build and install the Python library for you.
+
 See [CONTRIBUTING.md](../../CONTRIBUTING.md) for details on the
 general build process.