Skip to content

Upgrade documentation techstack (#6470) #1

Upgrade documentation techstack (#6470)

Upgrade documentation techstack (#6470) #1

Workflow file for this run

name: Windows
on:
workflow_dispatch:
inputs:
developer_build:
description: "Set to OFF for Release wheels"
required: false
default: "ON"
push:
branches:
- master
pull_request:
types: [opened, reopened, synchronize] # Rebuild on new pushes to PR
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
PIP_VER: "23.2.1"
WHEEL_VER: "0.38.4"
STOOLS_VER: "67.3.2"
JEDI_VER: "0.17.2" # https://github.com/ipython/ipython/issues/12740
IDNA_VER: "2.8" # https://github.com/psf/requests/issues/5710
SRC_DIR: "D:\\a\\open3d\\open3d"
BUILD_DIR: "C:\\Open3D\\build"
NPROC: 2
DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }}
GCE_CLI_GHA_VERSION: '416.0.0' # Fixed to avoid dependency on API changes
jobs:
windows:
runs-on: windows-2019
strategy:
fail-fast: false
matrix:
BUILD_SHARED_LIBS: [ON, OFF]
STATIC_RUNTIME: [ON, OFF]
BUILD_CUDA_MODULE: [ON, OFF]
CONFIG: [Release, Debug]
exclude:
- BUILD_SHARED_LIBS: ON
STATIC_RUNTIME: ON
include:
- BUILD_CUDA_MODULE: ON
CUDA_VERSION: 11.0.3
env:
BUILD_WEBRTC: ${{ ( matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' ) && 'ON' || 'OFF' }}
steps:
- name: Disk space used
run: Get-PSDrive
- name: Setup Windows SDK
uses: GuillaumeFalourd/[email protected]
with:
sdk-version: 19041
- name: Install CUDA
if: ${{ matrix.BUILD_CUDA_MODULE == 'ON' }}
run: |
# Define variables
$CUDA_VER_FULL = "${{ matrix.CUDA_VERSION }}"
$CUDA_VER_ARR = $CUDA_VER_FULL.Split(".")
$CUDA_VER = "$($CUDA_VER_ARR[0]).$($CUDA_VER_ARR[1])"
$CUDA_VER_ID = "$($CUDA_VER_ARR[0])_$($CUDA_VER_ARR[1])"
# Installer url
if ( $CUDA_VER_ARR[0] -ge 11 ) {
$CUDA_URL = "http://developer.download.nvidia.com/compute/cuda/$CUDA_VER_FULL/network_installers/cuda_$($CUDA_VER_FULL)_win10_network.exe"
} else {
$CUDA_URL = "http://developer.download.nvidia.com/compute/cuda/$CUDA_VER/Prod/network_installers/cuda_$($CUDA_VER_FULL)_win10_network.exe"
}
# Installer arguments
$CUDA_INSTALL_ARGS = "-s"
# Required packages
$CUDA_PACKAGES = "nvcc", "visual_studio_integration", "cublas", "cublas_dev", "cudart", "cusolver", "cusolver_dev", "npp", "npp_dev"
$CUDA_PACKAGES.ForEach({ $CUDA_INSTALL_ARGS += " $($_)_$($CUDA_VER)" })
# Download and install CUDA
echo "Downloading CUDA installer from $CUDA_URL"
Invoke-WebRequest $CUDA_URL -OutFile cuda.exe
echo "Installing CUDA..."
Start-Process -Wait -FilePath .\cuda.exe -ArgumentList "$CUDA_INSTALL_ARGS"
if ( !$? ) {
exit 1
}
# Add CUDA environment variables.
$CUDA_PATH = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$CUDA_VER"
echo "CUDA_PATH=$CUDA_PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
echo "CUDA_PATH_V$CUDA_VER_ID=$CUDA_PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
echo "$CUDA_PATH\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
- name: Checkout source code
uses: actions/checkout@v3
- name: Set up Python version
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Config
# Move build directory to C: https://github.com/actions/virtual-environments/issues/1341
run: |
$ErrorActionPreference = 'Stop'
New-Item -Path ${{ env.BUILD_DIR }} -ItemType Directory
cd ${{ env.BUILD_DIR }}
if (${env:DEVELOPER_BUILD} -ne "OFF") {
${env:DEVELOPER_BUILD}="ON"
}
cmake -G "Visual Studio 16 2019" -A x64 `
-DDEVELOPER_BUILD="${env:DEVELOPER_BUILD}" `
-DCMAKE_SYSTEM_VERSION="10.0.19041.0" `
-DBUILD_EXAMPLES=OFF `
-DCMAKE_INSTALL_PREFIX="C:\Program Files\Open3D" `
-DBUILD_SHARED_LIBS=${{ matrix.BUILD_SHARED_LIBS }} `
-DSTATIC_WINDOWS_RUNTIME=${{ matrix.STATIC_RUNTIME }} `
-DBUILD_COMMON_ISPC_ISAS=ON `
-DBUILD_LIBREALSENSE=ON `
-DBUILD_WEBRTC=${{ env.BUILD_WEBRTC }} `
-DBUILD_UNIT_TESTS=ON `
-DBUILD_CUDA_MODULE=${{ matrix.BUILD_CUDA_MODULE }} `
${{ env.SRC_DIR }}
- name: Build
working-directory: ${{ env.BUILD_DIR }}
run: |
$ErrorActionPreference = 'Stop'
cmake --build . --parallel ${{ env.NPROC }} --config ${{ matrix.CONFIG }} `
--target build-examples-iteratively
cmake --build . --parallel ${{ env.NPROC }} --config ${{ matrix.CONFIG }} `
--target INSTALL
- name: Package
working-directory: ${{ env.BUILD_DIR }}
if: ${{ matrix.BUILD_SHARED_LIBS == 'ON' && matrix.BUILD_CUDA_MODULE == 'OFF' }}
run: |
$ErrorActionPreference = 'Stop'
cmake --build . --parallel ${{ env.NPROC }} --config ${{ matrix.CONFIG }} `
--target package
if ("${{ matrix.CONFIG }}" -eq "Debug") {
Get-ChildItem package/open3d-devel-*.zip | Rename-Item -NewName `
{$_.name -Replace '.zip','-dbg.zip'}
}
$DEVEL_PKG_NAME=(Get-ChildItem package/open3d-devel-*.zip).Name
echo "DEVEL_PKG_NAME=$DEVEL_PKG_NAME" | Out-File -FilePath `
$env:GITHUB_ENV -Encoding utf8 -Append
- name: Upload Package
if: ${{ matrix.BUILD_SHARED_LIBS == 'ON' && matrix.BUILD_CUDA_MODULE == 'OFF' }}
uses: actions/upload-artifact@v3
with:
name: open3d-devel-windows
path: ${{ env.BUILD_DIR }}/package/${{ env.DEVEL_PKG_NAME }}
if-no-files-found: error
- name: Viewer App
working-directory: ${{ env.BUILD_DIR }}
if: ${{ matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' && matrix.BUILD_CUDA_MODULE == 'OFF' && matrix.CONFIG == 'Release' }}
run: |
$ErrorActionPreference = 'Stop'
cmake --build . --parallel ${{ env.NPROC }} --config ${{ matrix.CONFIG }} `
--target Open3DViewer
cmake --build . --parallel ${{ env.NPROC }} --config ${{ matrix.CONFIG }} `
--target INSTALL
- name: Upload Viewer
if: ${{ matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' && matrix.BUILD_CUDA_MODULE == 'OFF' && matrix.CONFIG == 'Release' }}
uses: actions/upload-artifact@v3
with:
name: open3d-app-windows-amd64
path: C:\Program Files\Open3D\bin\Open3D
if-no-files-found: error
- name: GCloud CLI auth
if: ${{ github.ref == 'refs/heads/master' && matrix.BUILD_SHARED_LIBS == 'ON' && matrix.BUILD_CUDA_MODULE == 'OFF' }}
uses: google-github-actions/auth@v1
with:
project_id: ${{ secrets.GCE_PROJECT }}
credentials_json: '${{ secrets.GCE_SA_KEY_GPU_CI }}'
- name: GCloud CLI setup
if: ${{ github.ref == 'refs/heads/master' && matrix.BUILD_SHARED_LIBS == 'ON' && matrix.BUILD_CUDA_MODULE == 'OFF' }}
uses: google-github-actions/setup-gcloud@v1
with:
version: ${{ env.GCE_CLI_GHA_VERSION }}
project_id: ${{ secrets.GCE_DOCS_PROJECT }}
- name: Upload package to GCS bucket
if: ${{ github.ref == 'refs/heads/master' && matrix.BUILD_SHARED_LIBS == 'ON' && matrix.BUILD_CUDA_MODULE == 'OFF' }}
run: |
gsutil cp ${{ env.BUILD_DIR }}/package/${{ env.DEVEL_PKG_NAME }} `
gs://open3d-releases-master/devel/
if ($LastExitCode -eq 0) {
echo "Download devel package at: https://storage.googleapis.com/open3d-releases-master/devel/${{ env.DEVEL_PKG_NAME }}"
} else {
throw "Devel package upload failed"
}
- name: Run C++ unit tests
if: ${{ matrix.BUILD_CUDA_MODULE == 'OFF' }}
working-directory: ${{ env.BUILD_DIR }}
run: |
echo "Add --gtest_random_seed=SEED to the test command to repeat this test sequence."
.\bin\${{ matrix.CONFIG }}\tests.exe --gtest_shuffle --gtest_filter=-*ReduceSum64bit2DCase0*:*ReduceSum64bit2DCase3*
- name: Linking to Open3D
run: |
$ErrorActionPreference = 'Stop'
git clone https://github.com/isl-org/open3d-cmake-find-package.git
cd open3d-cmake-find-package
mkdir build
cd build
cmake -G "Visual Studio 16 2019" -A x64 `
-DCMAKE_INSTALL_PREFIX="C:\Program Files\Open3D" `
-DSTATIC_WINDOWS_RUNTIME=${{ matrix.STATIC_RUNTIME }} `
..
cmake --build . --config ${{ matrix.CONFIG }}
.\${{ matrix.CONFIG }}\Draw.exe --skip-for-unit-test
Remove-Item "C:\Program Files\Open3D" -Recurse
- name: Install Python package
working-directory: ${{ env.BUILD_DIR }}
run: |
$ErrorActionPreference = 'Stop'
python -m pip install --upgrade pip==${{ env.PIP_VER }} `
wheel==${{ env.WHEEL_VER }} `
setuptools==${{ env.STOOLS_VER }} `
jedi==${{ env.JEDI_VER }} `
idna==${{ env.IDNA_VER }}
cmake --build . --config ${{ matrix.CONFIG }} --target install-pip-package
- name: Import python package
# If BUILD_SHARED_LIBS == ON, Open3D.dll needs to be copied, which is not recommended for python.
if: ${{ matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.BUILD_CUDA_MODULE == 'OFF' }}
run: |
python -c "import open3d; print('Imported:', open3d)"
python -c "import open3d; print('CUDA enabled: ', open3d.core.cuda.is_available())"
- name: Disk space used
run: Get-PSDrive
build-wheel:
name: Build wheel
runs-on: windows-2019
strategy:
fail-fast: false
# https://github.community/t/how-to-conditionally-include-exclude-items-in-matrix-eg-based-on-branch/16853/6
matrix:
python_version: ['3.8', '3.9', '3.10', '3.11']
is_master:
- ${{ github.ref == 'refs/heads/master' }}
exclude:
- is_master: false
python_version: '3.8'
- is_master: false
python_version: '3.9'
- is_master: false
python_version: '3.10'
steps:
- name: Checkout source code
uses: actions/checkout@v3
- name: Setup Windows SDK
uses: GuillaumeFalourd/[email protected]
with:
sdk-version: 19041
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python_version }}
- name: Install Python dependencies
working-directory: ${{ env.SRC_DIR }}
run: |
$ErrorActionPreference = 'Stop'
python -m pip install -r python/requirements.txt
python -m pip install -r python/requirements_jupyter_build.txt
- name: Config
run: |
$ErrorActionPreference = 'Stop'
New-Item -Path ${{ env.BUILD_DIR }} -ItemType Directory
cd ${{ env.BUILD_DIR }}
if (${env:DEVELOPER_BUILD} -ne "OFF") {
${env:DEVELOPER_BUILD}="ON"
}
cmake -G "Visual Studio 16 2019" -A x64 `
-DCMAKE_INSTALL_PREFIX="C:\Program Files\Open3D" `
-DDEVELOPER_BUILD="${env:DEVELOPER_BUILD}" `
-DCMAKE_SYSTEM_VERSION="10.0.19041.0" `
-DBUILD_SHARED_LIBS=OFF `
-DSTATIC_WINDOWS_RUNTIME=ON `
-DBUILD_COMMON_ISPC_ISAS=ON `
-DBUILD_AZURE_KINECT=ON `
-DBUILD_LIBREALSENSE=ON `
-DBUILD_WEBRTC=ON `
-DBUILD_JUPYTER_EXTENSION=ON `
${{ env.SRC_DIR }}
- name: Build Python package
working-directory: ${{ env.BUILD_DIR }}
run: |
$ErrorActionPreference = 'Stop'
python -m pip install --upgrade pip==${{ env.PIP_VER }} `
wheel==${{ env.WHEEL_VER }} `
setuptools==${{ env.STOOLS_VER }}
cmake --build . --parallel ${{ env.NPROC }} --config Release --target pip-package
$PIP_PKG_NAME=(Get-ChildItem lib/python_package/pip_package/open3d*.whl).Name
echo "PIP_PKG_NAME=$PIP_PKG_NAME" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
- name: Upload wheel
uses: actions/upload-artifact@v3
with:
name: open3d_win_amd64_wheels
path: ${{ env.BUILD_DIR }}/lib/python_package/pip_package/${{ env.PIP_PKG_NAME }}
if-no-files-found: error
- name: GCloud CLI auth
if: ${{ github.ref == 'refs/heads/master' }}
uses: google-github-actions/auth@v1
with:
project_id: ${{ secrets.GCE_PROJECT }}
credentials_json: '${{ secrets.GCE_SA_KEY_GPU_CI }}'
- name: GCloud CLI setup
if: ${{ github.ref == 'refs/heads/master' }}
uses: google-github-actions/setup-gcloud@v1
with:
version: ${{ env.GCE_CLI_GHA_VERSION }}
project_id: ${{ secrets.GCE_DOCS_PROJECT }}
- name: Upload wheel to GCS bucket
if: ${{ github.ref == 'refs/heads/master' }}
env:
python_version: ${{ matrix.python_version }}
run: |
$ErrorActionPreference = 'Stop'
$PYTAG="-cp$(${{ env.python_version }} -replace '\.', '')"
gsutil cp ${{ env.BUILD_DIR }}/lib/python_package/pip_package/${{ env.PIP_PKG_NAME }} gs://open3d-releases-master/python-wheels/
if ($LastExitCode -eq 0) {
echo "Download pip package at: https://storage.googleapis.com/open3d-releases-master/python-wheels/${{ env.PIP_PKG_NAME }}"
} else {
throw "Wheel upload failed"
}
test-wheel:
name: Test wheel
runs-on: windows-2019
needs: [build-wheel]
strategy:
fail-fast: false
matrix:
python_version: ['3.8', '3.9', '3.10', '3.11']
is_master:
- ${{ github.ref == 'refs/heads/master' }}
exclude:
- is_master: false
python_version: '3.8'
- is_master: false
python_version: '3.9'
- is_master: false
python_version: '3.10'
steps:
- name: Checkout source code
uses: actions/checkout@v3
- name: Download wheels
uses: actions/download-artifact@v3
# See https://github.com/dawidd6/action-download-artifact for more
# flexible artifact download options
with:
name: open3d_win_amd64_wheels
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python_version }}
- name: Test Python package
run: |
$ErrorActionPreference = 'Stop'
python -V
python -m venv open3d_test_venv
open3d_test_venv\Scripts\Activate.ps1
python -m pip install --upgrade pip==${{ env.PIP_VER }} `
wheel==${{ env.WHEEL_VER }} `
setuptools==${{ env.STOOLS_VER }}
python -m pip install -U -r python/requirements_test.txt
$py_tag=(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')")
if (Test-Path -Path "pip_package") {
$PIP_PKG_NAME=(Get-ChildItem pip_package\open3d*-$py_tag-*.whl).Name
} else {
$PIP_PKG_NAME=(Get-ChildItem open3d*-$py_tag-*.whl).Name
}
echo "Installing Open3D wheel $PIP_PKG_NAME in virtual environment..."
python -m pip install "$PIP_PKG_NAME"
python -c "import open3d; print('Imported:', open3d)"
python -c "import open3d; print('CUDA enabled: ', open3d.core.cuda.is_available())"
deactivate
- name: Run Python unit tests
run: |
$ErrorActionPreference = 'Stop'
open3d_test_venv\Scripts\Activate.ps1
echo "Running Open3D python tests..."
echo "Add --randomly-seed=SEED to the test command to reproduce test order."
echo "Testing ML and ML Ops disabled"
python -m pytest python/test/ --ignore python/test/ml_ops/
deactivate
ready-docs:
name: Ready docs archive
# no need to run on Windows
runs-on: ubuntu-latest
if: ${{ github.ref == 'refs/heads/master' }}
# temp workaround for Windows CUDA Debug CI out of space. Still update docs.
# needs: [build-wheel, windows]
needs: [build-wheel]
steps:
- name: GCloud CLI auth
uses: google-github-actions/auth@v1
with:
project_id: ${{ secrets.GCE_PROJECT }}
credentials_json: '${{ secrets.GCE_SA_KEY_GPU_CI }}'
- name: GCloud CLI setup
uses: google-github-actions/setup-gcloud@v1
with:
version: ${{ env.GCE_CLI_GHA_VERSION }}
project_id: ${{ secrets.GCE_PROJECT }}
- name: Check wheels and ready documentation archive
run: |
touch marker_file
gsutil cp marker_file gs://open3d-docs/${{ github.sha }}_ready_windows
if [ $(gsutil ls gs://open3d-docs/${{ github.sha }}_ready* | wc -l)\
-eq 4 ]; then
echo "All wheels and docs available. Making docs ready."
# Remove all marker files: Note _ at end of pattern.
gsutil rm gs://open3d-docs/${{ github.sha }}_ready_*
# Rename docs archive:
gsutil mv gs://open3d-docs/${{ github.sha }}_ready.tar.gz \
gs://open3d-docs/${{ github.sha }}.tar.gz
# Set holds on new artifacts, release on old
gsutil retention temp release gs://open3d-releases-master/*
gsutil retention temp set gs://open3d-releases-master/python-wheels/*${GITHUB_SHA:0:7}*.whl
gsutil retention temp set gs://open3d-releases-master/devel/*${GITHUB_SHA:0:7}*
else
echo "All wheels / docs not available yet."
fi