Skip to content

Upgrade documentation techstack (#6470) #1

Upgrade documentation techstack (#6470)

Upgrade documentation techstack (#6470) #1

Workflow file for this run

name: MacOS
on:
workflow_dispatch:
inputs:
developer_build:
description: 'Set to OFF for Release builds.'
required: false
default: 'ON'
push:
branches:
- master
pull_request:
types: [opened, reopened, synchronize] # Rebuild on new pushes to PR
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
NPROC: 4
DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }}
GCE_CLI_GHA_VERSION: '416.0.0' # Fixed to avoid dependency on API changes
jobs:
MacOS:
runs-on: macos-12
strategy:
fail-fast: false
matrix:
CONFIG: [ON, OFF]
env:
BUILD_SHARED_LIBS: ${{ matrix.CONFIG }}
BUILD_CUDA_MODULE: OFF
BUILD_TENSORFLOW_OPS: ${{ matrix.CONFIG }}
BUILD_PYTORCH_OPS: ${{ matrix.CONFIG }}
LOW_MEM_USAGE: ON
steps:
- name: Checkout source code
uses: actions/checkout@v3
- name: Setup cache
uses: actions/cache@v3
with:
# Ref: https://github.com/apache/incubator-mxnet/pull/18459/files
path: ~/.ccache
# We include the commit sha in the cache key, as new cache entries are
# only created if there is no existing entry for the key yet.
key: ${{ runner.os }}-ccache-${{ github.sha }}
# Restore any ccache cache entry, if none for
# ${{ runner.os }}-ccache-${{ github.sha }} exists.
# Common prefix will be used so that ccache can be used across commits.
restore-keys: |
${{ runner.os }}-ccache
- name: Set up Python version
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Install dependencies
run: |
brew install ccache pkg-config
# Install libomp 11.1.0 from old brew bottle for catalina (10.15).
# Directly installing the Ruby formula will install for the current OS
# https://github.com/microsoft/LightGBM/issues/4229
brew unlink libomp
curl -L -H "Authorization: Bearer QQ==" -o libomp-11.1.0.catalina.bottle.tar.gz \
https://ghcr.io/v2/homebrew/core/libomp/blobs/sha256:45a5aa653bd45bd5ff5858580b1a4670c4b5a51ea29d68d45a53f72f56010e05
brew install -f libomp-11.1.0.catalina.bottle.tar.gz
ccache -M 2G # See .github/workflows/readme.md for ccache strategy.
- name: Config and build
run: |
PATH=/usr/local/var/homebrew/linked/ccache/libexec:$PATH
ccache -s
./util/run_ci.sh
DEVEL_PKG_NAME="$(basename package/open3d-devel-*.tar.xz)"
echo "DEVEL_PKG_NAME=$DEVEL_PKG_NAME" >> $GITHUB_ENV
- name: Build Open3D viewer app
if: ${{ env.BUILD_SHARED_LIBS == 'OFF' }}
run: |
PATH=/usr/local/var/homebrew/linked/ccache/libexec:$PATH
pushd build
make -j${NPROC} Open3DViewer
pushd bin
zip -rv open3d-app-macosx-10_15.zip Open3D.app
ccache -s
- name: Upload package
if: ${{ env.BUILD_SHARED_LIBS == 'ON' }}
uses: actions/upload-artifact@v3
with:
name: open3d-devel-macosx
path: build/package/${{ env.DEVEL_PKG_NAME }}
if-no-files-found: error
- name: GCloud CLI auth
if: ${{ github.ref == 'refs/heads/master' && env.BUILD_SHARED_LIBS == 'ON' }}
uses: 'google-github-actions/auth@v1'
with:
project_id: ${{ secrets.GCE_PROJECT }}
credentials_json: '${{ secrets.GCE_SA_KEY_GPU_CI }}'
- name: GCloud CLI setup
if: ${{ github.ref == 'refs/heads/master' && env.BUILD_SHARED_LIBS == 'ON' }}
uses: google-github-actions/setup-gcloud@v1
with:
version: ${{ env.GCE_CLI_GHA_VERSION }}
project_id: ${{ secrets.GCE_PROJECT }}
- name: Upload package to GCS bucket
if: ${{ github.ref == 'refs/heads/master' && env.BUILD_SHARED_LIBS == 'ON' }}
run: |
gsutil cp build/package/${{ env.DEVEL_PKG_NAME }} gs://open3d-releases-master/devel/
echo "Download devel package at: https://storage.googleapis.com/open3d-releases-master/devel/${{ env.DEVEL_PKG_NAME }}"
- name: Upload Open3D viewer app
uses: actions/upload-artifact@v3
if: ${{ env.BUILD_SHARED_LIBS == 'OFF' }}
with:
name: open3d-app-macosx-10_15
path: build/bin/open3d-app-macosx-10_15.zip
if-no-files-found: error
build-wheel:
name: Build wheel
runs-on: macos-12
strategy:
fail-fast: false
# https://github.community/t/how-to-conditionally-include-exclude-items-in-matrix-eg-based-on-branch/16853/6
matrix:
python_version: ['3.8', '3.9', '3.10', '3.11']
is_master:
- ${{ github.ref == 'refs/heads/master' }}
exclude:
- is_master: false
python_version: '3.8'
- is_master: false
python_version: '3.9'
- is_master: false
python_version: '3.10'
env:
BUILD_CUDA_MODULE: OFF
OPEN3D_ML_ROOT: ${{ github.workspace }}/Open3D-ML
steps:
- name: Checkout source code
uses: actions/checkout@v3
- name: Checkout Open3D-ML source code
uses: actions/checkout@v3
with:
repository: isl-org/Open3D-ML
path: ${{ env.OPEN3D_ML_ROOT }}
- name: Setup cache
uses: actions/cache@v3
with:
# Ref: https://github.com/apache/incubator-mxnet/pull/18459/files
path: ~/.ccache
# We include the commit sha in the cache key, as new cache entries are
# only created if there is no existing entry for the key yet.
key: ${{ runner.os }}-ccache-${{ github.sha }}
# Restore any ccache cache entry, if none for
# ${{ runner.os }}-ccache-${{ github.sha }} exists.
# Common prefix will be used so that ccache can be used across commits.
restore-keys: |
${{ runner.os }}-ccache
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python_version }}
- name: Install dependencies
run: |
echo "Using python: $(which python)"
python --version
echo -n "Using pip: $(which pip)"
python -m pip --version
echo "Using cmake: $(which cmake)"
cmake --version
source util/ci_utils.sh
install_python_dependencies
# Install libomp 11.1.0. See comment above.
brew install ccache
brew unlink libomp
curl -L -H "Authorization: Bearer QQ==" -o libomp-11.1.0.catalina.bottle.tar.gz \
https://ghcr.io/v2/homebrew/core/libomp/blobs/sha256:45a5aa653bd45bd5ff5858580b1a4670c4b5a51ea29d68d45a53f72f56010e05
brew install -f libomp-11.1.0.catalina.bottle.tar.gz
ccache -M 2G # See .github/workflows/readme.md for ccache strategy.
- name: Config and build wheel
run: |
PATH=/usr/local/var/homebrew/linked/ccache/libexec:$PATH
ccache -s
source util/ci_utils.sh
build_pip_package
ccache -s
PIP_PKG_NAME="$(basename build/lib/python_package/pip_package/open3d*.whl)"
echo "PIP_PKG_NAME=$PIP_PKG_NAME" >> $GITHUB_ENV
- name: Upload wheel
uses: actions/upload-artifact@v3
with:
name: open3d_macosx_x86_64_wheels
path: build/lib/python_package/pip_package/${{ env.PIP_PKG_NAME }}
if-no-files-found: error
- name: GCloud CLI auth
if: ${{ github.ref == 'refs/heads/master' }}
uses: 'google-github-actions/auth@v1'
with:
project_id: ${{ secrets.GCE_PROJECT }}
credentials_json: '${{ secrets.GCE_SA_KEY_GPU_CI }}'
- name: GCloud CLI setup
if: ${{ github.ref == 'refs/heads/master' }}
uses: google-github-actions/setup-gcloud@v1
with:
version: ${{ env.GCE_CLI_GHA_VERSION }}
project_id: ${{ secrets.GCE_PROJECT }}
- name: Upload wheel to GCS bucket
if: ${{ github.ref == 'refs/heads/master' }}
env:
python_version: ${{ matrix.python_version }}
run: |
PYTAG="-cp$(echo ${{ env.python_version }} | tr -d '.')"
gsutil cp build/lib/python_package/pip_package/${{ env.PIP_PKG_NAME }} gs://open3d-releases-master/python-wheels/
echo "Download pip package at: https://storage.googleapis.com/open3d-releases-master/python-wheels/${{ env.PIP_PKG_NAME }}"
test-wheel:
name: Test wheel
runs-on: macos-12
needs: [build-wheel]
strategy:
fail-fast: false
matrix:
python_version: ['3.8', '3.9', '3.10', '3.11']
is_master:
- ${{ github.ref == 'refs/heads/master' }}
exclude:
- is_master: false
python_version: '3.8'
- is_master: false
python_version: '3.9'
- is_master: false
python_version: '3.10'
env:
OPEN3D_ML_ROOT: ${{ github.workspace }}/Open3D-ML
steps:
- name: Checkout source code
uses: actions/checkout@v3
- name: Checkout Open3D-ML source code
uses: actions/checkout@v3
with:
repository: isl-org/Open3D-ML
path: ${{ env.OPEN3D_ML_ROOT }}
- name: Download wheels
uses: actions/download-artifact@v3
# See https://github.com/dawidd6/action-download-artifact for more
# flexible artifact download options
with:
name: open3d_macosx_x86_64_wheels
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python_version }}
- name: Test Python package
run: |
python -V
source util/ci_utils.sh
pi_tag=$(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')")
test_wheel open3d*-"$pi_tag"-*.whl
- name: Run Python unit tests (benchmarks)
run: |
source util/ci_utils.sh
echo "Running Open3D python tests..."
run_python_tests
ready-docs:
name: Ready docs archive
# no need to run on macOS
runs-on: ubuntu-latest
if: ${{ github.ref == 'refs/heads/master' }}
needs: [build-wheel, MacOS]
steps:
- name: GCloud CLI auth
uses: 'google-github-actions/auth@v1'
with:
project_id: ${{ secrets.GCE_PROJECT }}
credentials_json: '${{ secrets.GCE_SA_KEY_GPU_CI }}'
- name: GCloud CLI setup
uses: google-github-actions/setup-gcloud@v1
with:
version: ${{ env.GCE_CLI_GHA_VERSION }}
project_id: ${{ secrets.GCE_PROJECT }}
- name: Check wheels and ready documentation archive
run: |
touch marker_file
gsutil cp marker_file gs://open3d-docs/${{ github.sha }}_ready_macos
if [ $(gsutil ls gs://open3d-docs/${{ github.sha }}_ready* | wc -l)\
-eq 4 ]; then
echo "All wheels and docs available. Making docs ready."
# Remove all marker files: Note _ at end of pattern.
gsutil rm gs://open3d-docs/${{ github.sha }}_ready_*
# Rename docs archive:
gsutil mv gs://open3d-docs/${{ github.sha }}_ready.tar.gz \
gs://open3d-docs/${{ github.sha }}.tar.gz
# Set holds on new artifacts, release on old
gsutil retention temp release gs://open3d-releases-master/*
gsutil retention temp set gs://open3d-releases-master/python-wheels/*${GITHUB_SHA:0:7}*.whl
gsutil retention temp set gs://open3d-releases-master/devel/*${GITHUB_SHA:0:7}*
else
echo "All wheels / docs not available yet. Docs not ready."
fi