Enable multithreading in synapse python client #262
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # build and test synapser. Additionally deploys to S3 RAN server on GitHub release. | |
| name: build | |
| on: | |
| push: | |
| # we build/test all pushed branches, but not tags. | |
| # we only push tags with releases, and we handle releases explicitly | |
| branches: | |
| - '**' | |
| tags-ignore: | |
| - '**' | |
| pull_request: | |
| release: | |
| types: | |
| - 'published' | |
| jobs: | |
| build: | |
| runs-on: ${{ matrix.os }} | |
| outputs: | |
| package_version: ${{ steps.shared-env.outputs.package_version }} | |
| r_minor_version: ${{ steps.shared-env.outputs.r_minor_version }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-22.04, macos-13, windows-2022] | |
| r: [4.1.3, 4.2.3, 4.3.1, 4.4.1] | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python Version | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: setup-r | |
| uses: r-lib/actions/setup-r@v2 | |
| with: | |
| r-version: ${{ matrix.r }} | |
| architecture: 'x64' | |
| - name: shared-env | |
| id: shared-env | |
| shell: bash | |
| run: | | |
| PACKAGE_NAME=synapser | |
| echo "PACKAGE_NAME=$PACKAGE_NAME" >> $GITHUB_ENV | |
| PACKAGE_VERSION=$(grep "Version: " DESCRIPTION | awk '{print $2'}) | |
| # if this was triggered by a release and the release tag | |
| # looks like semver then we use that for the package version. | |
| RELEASE_TAG=${{ github.event.release.tag_name }} | |
| if [[ $RELEASE_TAG =~ ^v?([[:digit:]\.]+)(-rc)? ]]; then | |
| RELEASE_VERSION="${BASH_REMATCH[1]}.$GITHUB_RUN_NUMBER" | |
| # a release version overrides the package version | |
| # for purposes of creating artifacts | |
| PACKAGE_VERSION=$RELEASE_VERSION | |
| DATE=`date +%Y-%m-%d` | |
| # replace DESCRIPTION with $VERSION & $DATE | |
| # sed -i not portable on OSX so we wash through some temp files instead | |
| sed "s|^Version: .*$|Version: $PACKAGE_VERSION|g" DESCRIPTION > DESCRIPTION.temp | |
| sed "s|^Date: .*$|Date: $DATE|g" DESCRIPTION.temp > DESCRIPTION2.temp | |
| rm DESCRIPTION | |
| mv DESCRIPTION2.temp DESCRIPTION | |
| rm DESCRIPTION.temp | |
| fi | |
| echo "BRANCH_VERSION=$BRANCH_VERSION" >> $GITHUB_ENV | |
| echo "PACKAGE_VERSION=$PACKAGE_VERSION" >> $GITHUB_ENV | |
| R_VERSION=${{matrix.r}} | |
| R_MAJOR_VERSION=$(echo $R_VERSION | cut -f1 -d".") | |
| R_MINOR_VERSION=$R_MAJOR_VERSION.$(echo $R_VERSION | cut -f2 -d".") | |
| echo "R_VERSION=$R_VERSION" >> $GITHUB_ENV | |
| echo "R_MAJOR_VERSION=$R_MAJOR_VERSION" >> $GITHUB_ENV | |
| echo "R_MINOR_VERSION=$R_MINOR_VERSION" >> $GITHUB_ENV | |
| # replace backslashes with forward slashes for windows. | |
| # windows bash will handle forward slashes fine and this | |
| # makes it easier to unify the path handling. | |
| R_LIBS_USER_SANITIZED=$(echo $R_LIBS_USER | sed 's/\\/\//g') | |
| echo "R_LIBS_USER=$R_LIBS_USER_SANITIZED" >> $GITHUB_ENV | |
| # echo "::set-output name=package_version::$PACKAGE_VERSION" | |
| # echo "::set-output name=r_minor_version::$R_MINOR_VERSION" | |
| echo "package_version=$PACKAGE_VERSION" >> $GITHUB_OUTPUT | |
| echo "r_minor_version=$R_MINOR_VERSION" >> $GITHUB_OUTPUT | |
| - name: setup-r-mac-13 | |
| if: ${{matrix.os == 'macos-13'}} | |
| run: | | |
| R_LIBS_USER=${GITHUB_WORKSPACE}/R_LIBS | |
| rm -rf R_LIBS_USER | |
| mkdir -p $R_LIBS_USER | |
| echo "R_LIBS_USER=$R_LIBS_USER" >> $GITHUB_ENV | |
| echo "R=R" >> $GITHUB_ENV | |
| - name: install-test-config | |
| shell: bash | |
| run: | | |
| if [ -z "${{ secrets.encrypted_d17283647768_key }}" ] || [ -z "${{ secrets.encrypted_d17283647768_key }}" ]; then | |
| echo "No test configuration decryption keys available, skipping integration tests" | |
| exit 1 | |
| fi | |
| # decrypt the encrypted test synapse configuration | |
| openssl aes-256-cbc -K ${{ secrets.encrypted_d17283647768_key }} -iv ${{ secrets.encrypted_d17283647768_iv }} -in test.synapseConfig.enc -out test.synapseConfig -d | |
| mv test.synapseConfig ~/.synapseConfig | |
| - name: install-pandoc | |
| uses: r-lib/actions/setup-pandoc@v2 | |
| - name: install-python-dependencies | |
| shell: bash | |
| if: ${{runner.os == 'Windows'}} | |
| run: | | |
| pip install 'synapseclient[pandas]' | |
| - uses: r-lib/actions/setup-r-dependencies@v2 | |
| with: | |
| cache: true | |
| cache-version: 2 | |
| architecture: 'x64' | |
| - name: linux-build-package | |
| if: ${{runner.os == 'linux'}} | |
| run: | | |
| Rscript -e "install.packages('remotes'); remotes::install_version('reticulate', version = '1.28')" | |
| Rscript -e "reticulate::install_miniconda()" | |
| echo "options(reticulate.conda_binary = reticulate:::miniconda_conda())" >> .Rprofile | |
| Rscript -e "reticulate::conda_create('r-reticulate', packages = c('python==3.10'))" | |
| R CMD build ./ | |
| R CMD INSTALL ./ --library=$R_LIBS_USER --no-test-load | |
| echo "ARTIFACT_EXTENSION=tar.gz" >> $GITHUB_ENV | |
| - name: mac-build-package | |
| if: ${{runner.os == 'macOS'}} | |
| run: | | |
| Rscript -e "install.packages('remotes'); remotes::install_version('reticulate', version = '1.28')" | |
| Rscript -e "reticulate::install_miniconda()" | |
| echo "options(reticulate.conda_binary = reticulate:::miniconda_conda())" >> .Rprofile | |
| Rscript -e "reticulate::conda_create('r-reticulate', packages = c('python==3.10'))" | |
| R CMD build ./ | |
| R CMD INSTALL --build ${PACKAGE_NAME}_${PACKAGE_VERSION}.tar.gz --library=$R_LIBS_USER --no-test-load | |
| echo "ARTIFACT_EXTENSION=tgz" >> $GITHUB_ENV | |
| - name: windows-build-package | |
| if: ${{runner.os == 'Windows'}} | |
| shell: bash | |
| run: | | |
| echo "options(repos = c(CRAN = 'https://cloud.r-project.org'))" >> .Rprofile | |
| Rscript -e "install.packages('remotes'); remotes::install_version('reticulate', version = '1.28')" | |
| Rscript -e "reticulate::install_python(version = '3.10.11')" | |
| Rscript -e "reticulate::virtualenv_create(envname='r-reticulate',version = '3.10.11')" | |
| Rscript -e "reticulate::use_virtualenv('r-reticulate')" | |
| R CMD build ./ | |
| R CMD INSTALL --build ${PACKAGE_NAME}_${PACKAGE_VERSION}.tar.gz --library=$R_LIBS_USER --no-test-load --no-multiarch | |
| echo "ARTIFACT_EXTENSION=zip" >> $GITHUB_ENV | |
| - name: run-tests | |
| shell: bash | |
| run: | | |
| echo ".libPaths(c('$R_LIBS_USER', .libPaths()));" > runTests.R | |
| echo "setwd(sprintf('%s/tests', getwd()));" >> runTests.R | |
| echo "source('testthat.R')" >> runTests.R | |
| echo "library(synapser);" >> runTests.R | |
| echo "detach(\"package:synapser\", unload=TRUE);" >> runTests.R | |
| echo "library(synapser)" >> runTests.R | |
| R --vanilla < runTests.R | |
| rm runTests.R | |
| - name: artifact-name | |
| shell: bash | |
| run: | | |
| # we format our artifact names so we align to the same pattern | |
| # used by the deploy gist written to work with jenkins. | |
| ARTIFACT_NAME="${PACKAGE_NAME}_${PACKAGE_VERSION}.${ARTIFACT_EXTENSION}" | |
| OS_LABEL=$(echo ${{runner.os}} | tr '[:upper:]' '[:lower:]') | |
| if [[ "$OS_LABEL" == "macos" ]]; then | |
| OS_LABEL="mac" | |
| fi | |
| UPLOAD_NAME="label=${OS_LABEL}-RVERS-${{ steps.shared-env.outputs.r_minor_version }}" | |
| echo "ARTIFACT_NAME=$ARTIFACT_NAME" >> $GITHUB_ENV | |
| echo "UPLOAD_NAME=$UPLOAD_NAME" >> $GITHUB_ENV | |
| - name: upload-artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{env.UPLOAD_NAME}} | |
| path: ${{env.ARTIFACT_NAME}} | |
| deploy: | |
| runs-on: ubuntu-22.04 | |
| needs: build | |
| permissions: | |
| id-token: write | |
| contents: read | |
| if: github.event_name == 'release' | |
| outputs: | |
| s3_ran: ${{ steps.deploy-to-target.outputs.s3_ran }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: check-deployment-target | |
| id: check-deployment-target | |
| if: ${{github.event.action == 'published'}} | |
| shell: bash | |
| run: | | |
| DEPLOY_TARGET="" | |
| RELEASE_VERSION="" | |
| RELEASE_TAG=${{ github.event.release.tag_name }} | |
| if [[ $RELEASE_TAG =~ ^v?([[:digit:]\.]+)(-rc)? ]]; then | |
| RELEASE_VERSION="${BASH_REMATCH[1]}" | |
| echo $RELEASE_VERSION | |
| if [[ -n "$RELEASE_VERSION" ]]; then | |
| if [[ "${{ github.event.release.prerelease }}" == "true" ]]; then | |
| DEPLOY_TARGET="staging" | |
| else | |
| DEPLOY_TARGET="prod" | |
| fi | |
| fi | |
| fi | |
| echo "DEPLOY_TARGET=$DEPLOY_TARGET" >> $GITHUB_ENV | |
| # echo "::set-output name=deploy_target::$DEPLOY_TARGET" | |
| echo "deploy_target=$DEPLOY_TARGET" >> $GITHUB_OUTPUT | |
| - name: download-artifacts | |
| uses: actions/download-artifact@v4 | |
| if: ${{steps.check-deployment-target.outputs.deploy_target != ''}} | |
| with: | |
| path: deploy_artifacts | |
| # Before deoloying to RAN (which is an S3 bucket), authenticate to AWS using GitHub OIDC | |
| - name: Assume AWS Role | |
| uses: aws-actions/configure-aws-credentials@v3 | |
| with: | |
| aws-region: us-east-1 | |
| role-to-assume: arn:aws:iam::325565585839:role/sagebase-github-oidc-sage-bionetworks-synapser | |
| role-session-name: GitHubActions-${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }} | |
| role-duration-seconds: 1200 | |
| - name: deploy-to-target | |
| id: deploy-to-target | |
| if: ${{steps.check-deployment-target.outputs.deploy_target != ''}} | |
| shell: bash | |
| run: | | |
| # we use a gist to upload and the gist internally uses R so we need R on this runner. | |
| # TODO maybe create a docker image to run this with R preloaded (and/or an action). | |
| sudo apt-get -y update && sudo apt-get -y install r-base | |
| export ARTIFACTS_DIR=deploy_artifacts | |
| if [[ "$DEPLOY_TARGET" == "staging" ]]; then | |
| export S3_RAN=staging-ran.synapse.org | |
| elif [[ "$DEPLOY_TARGET" == "prod" ]]; then | |
| export S3_RAN=ran.synapse.org | |
| fi | |
| bash tools/deploy.sh | |
| # echo "::set-output name=s3_ran::$S3_RAN" | |
| echo "s3_ran=$S3_RAN" >> $GITHUB_OUTPUT | |
| # on each of our matrix platforms, download the newly uploaded package from RAN | |
| check-deploy: | |
| needs: | |
| [build, deploy] | |
| strategy: | |
| matrix: | |
| os: [ubuntu-22.04, windows-2022, macos-13] | |
| r: [4.1.3, 4.2.3, 4.3.1, 4.4.1] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: setup-r | |
| uses: r-lib/actions/setup-r@v2 | |
| with: | |
| r-version: ${{ matrix.r }} | |
| - name: setup-r-dependencies | |
| uses: r-lib/actions/setup-r-dependencies@v2 | |
| with: | |
| cache: true | |
| cache-version: 2 | |
| architecture: 'x64' | |
| - name: check-install | |
| shell: bash | |
| run: | | |
| RAN="http://${{ needs.deploy.outputs.s3_ran }}" | |
| VERSION_TO_CHECK="${{ needs.build.outputs.package_version }}" | |
| PACKAGE="synapser" | |
| echo "if (available.packages(repos='$RAN')['$PACKAGE','Version'] != '$VERSION_TO_CHECK') { quit(save = 'no', status = 1) }" > test.R | |
| echo "try(remove.packages('$PACKAGE'), silent=T)" >> test.R | |
| echo "install.packages('remotes', repos = 'https://cloud.r-project.org'); remotes::install_version('reticulate', version = '1.28', repos = 'https://cloud.r-project.org')" >> test.R | |
| echo "reticulate::install_python(version = '3.10.11')" >> test.R | |
| echo "reticulate::virtualenv_create(envname='r-reticulate',version = '3.10.11')" >> test.R | |
| echo "reticulate::use_virtualenv('r-reticulate')" >> test.R | |
| echo "install.packages('$PACKAGE', repos=c('$RAN', 'https://cloud.r-project.org/'))" >> test.R | |
| echo "library('$PACKAGE')" >> test.R | |
| R --vanilla < test.R |