Reorganization and cleanup of the repository #41
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Check Commits and Colab Images | |
| on: | |
| push: | |
| branches: [ "master" ] | |
| pull_request: | |
| branches: [ "master" ] | |
| workflow_dispatch: | |
| schedule: | |
| - cron: 0 12 */1 * * | |
| jobs: | |
| check_commits_and_images: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.11 | |
| - name: Install dependencies | |
| run: pip install requests==2.31.0 pandas==2.1.1 google-cloud-bigquery==3.12.0 pyarrow==13.0.0 nbformat==5.9.2 | |
| - name: Authorize Google Cloud | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| credentials_json: ${{ secrets.SERVICE_ACCOUNT_KEY }} | |
| create_credentials_file: true | |
| export_environment_variables: true | |
| - name: Run check-commits.py and check-colab-images.py, preprocess notebooks scripts | |
| run: | | |
| python test/src/check-commits.py | |
| python test/src/check-colab-images.py | |
| python test/src/preProcessNotebooks.py | |
| - name: Set result output | |
| id: set-result | |
| run: | | |
| if [[ -f "check_colab_images_result.txt" ]]; then | |
| RESULT=$(cat "check_colab_images_result.txt") | |
| echo "RESULT=$RESULT" >> $GITHUB_ENV | |
| fi | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| tool-cache: false | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: true | |
| docker-images: true | |
| swap-storage: true | |
| - name: Docker login | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Pull from GCP and Push Docker image to Docker Hub | |
| if: env.RESULT == 'true' | |
| run: | | |
| docker pull us-docker.pkg.dev/colab-images/public/runtime:latest | |
| docker tag us-docker.pkg.dev/colab-images/public/runtime:latest imagingdatacommons/idc-testing-colab:latest | |
| docker push imagingdatacommons/idc-testing-colab:latest | |
| - name: Pull Docker image from Docker Hub | |
| if: env.RESULT == 'false' | |
| run: | | |
| docker pull imagingdatacommons/idc-testing-colab:latest | |
| - name: Copy Google Cloud credentials to Docker container | |
| run: | | |
| CREDENTIALS_FILE_PATH="${{ env.GOOGLE_APPLICATION_CREDENTIALS }}" | |
| CREDENTIALS_FILE_NAME=$(basename "$CREDENTIALS_FILE_PATH") | |
| GOOGLE_APPLICATION_CREDENTIALS="/content/$CREDENTIALS_FILE_NAME" | |
| echo "GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS" >> $GITHUB_ENV | |
| - name: Run notebook with papermill | |
| run: | | |
| for nb in part1_prerequisites part2_searching_basics part3_exploring_cohorts; do | |
| docker run -d --name colab -v "$(pwd):/content" -e GOOGLE_APPLICATION_CREDENTIALS="${{ env.GOOGLE_APPLICATION_CREDENTIALS }}" imagingdatacommons/idc-testing-colab:latest | |
| docker exec -t colab /bin/bash -c "pip install papermill==2.4.0" | |
| docker exec -t colab /bin/bash -c "set -o xtrace && set -o errexit && set -o pipefail && set -o nounset && set +o errexit && cd content/ && papermill /content/notebooks/getting_started/${nb}.ipynb /content/test/outputs/${nb}_papermill_output.ipynb && set -o errexit && ls -A" | |
| #docker exec -t colab /bin/bash -c "jupyter nbconvert --to html --ExtractOutputPreprocessor.enabled=False /content/test/outputs/output_${nb}.ipynb" | |
| docker stop colab | |
| docker rm colab | |
| done | |
| # skip commit | |
| #- name: Commit changes | |
| # if: ${{ github.event_name != 'pull_request' }} | |
| # uses: stefanzweifel/git-auto-commit-action@v4 | |
| # with: | |
| # commit_message: 'Check colab env' | |
| # file_pattern: 'test/*.csv test/outputs/*.ipynb' | |
| # branch: 'master' | |
| #- name: Check output notebooks for errors | |
| # run: | | |
| # for nb in part1_prerequisites part2_searching_basics part3_exploring_cohorts; do | |
| # if grep -q '"name": "stderr"\|"status": "failed"' test/outputs/output_${nb}.ipynb; then | |
| # echo "Error messages found in the ${nb} notebook output:" | |
| # cat test/outputs/output_${nb}.ipynb | |
| # exit 1 | |
| # else | |
| # echo "No errors found in the ${nb} notebook output." | |
| # fi | |
| # done | |
| # exit $EXIT_CODE |