From 4e360ab8460334b34e258e7ba5dc49b9fc12863f Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 19 Nov 2024 12:57:52 -0800 Subject: [PATCH] Try alternative to checkout test test test test test test test test fix test test test test test test test test test test test test test test test test test test test test test test test test test test test fix test --- .github/actions/binary-upload/action.yml | 57 +++++++++++++++++++ .../actions/setup-binary-builds/action.yml | 16 ++++++ .github/workflows/_binary_upload.yml | 6 +- .github/workflows/build_wheels_linux.yml | 30 ++++++++-- 4 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 .github/actions/binary-upload/action.yml diff --git a/.github/actions/binary-upload/action.yml b/.github/actions/binary-upload/action.yml new file mode 100644 index 0000000000..b898c4bba6 --- /dev/null +++ b/.github/actions/binary-upload/action.yml @@ -0,0 +1,57 @@ +name: Binary upload action + +description: pull a specific docker image + +inputs: + repository: + description: 'Repository to checkout, defaults to ""' + default: '' + type: string + trigger-event: + description: "Trigger Event in caller that determines whether or not to upload" + type: string + default: '' + +runs: + using: composite + steps: + - name: Configure aws credentials (pytorch account) + if: ${{ inputs.trigger-event == 'schedule' || (inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly')) }} + uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: arn:aws:iam::749337293305:role/gha_workflow_nightly_build_wheels + aws-region: us-east-1 + + - name: Configure aws credentials (pytorch account) + if: ${{ env.CHANNEL == 'test' && startsWith(github.event.ref, 'refs/tags/v') }} + uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: arn:aws:iam::749337293305:role/gha_workflow_test_build_wheels + aws-region: us-east-1 + + - name: Nightly or release RC + if: ${{ inputs.trigger-event == 'schedule' || (inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly')) || (env.CHANNEL == 'test' && startsWith(github.event.ref, 'refs/tags/')) }} + shell: bash + run: | + set -ex + echo "NIGHTLY_OR_TEST=1" >> "${GITHUB_ENV}" + + - name: Upload package to pytorch.org + shell: bash + working-directory: ${{ inputs.repository }} + run: | + set -ex + + # shellcheck disable=SC1090 + source "${BUILD_ENV_FILE}" + + pip install awscli==1.32.18 + + AWS_CMD="aws s3 cp --dryrun" + if [[ "${NIGHTLY_OR_TEST:-0}" == "1" ]]; then + AWS_CMD="aws s3 cp" + fi + + for pkg in dist/*; do + ${AWS_CMD} "$pkg" "${PYTORCH_S3_BUCKET_PATH}" --acl public-read + done diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml index 52fbf36a20..f21281fbbe 100644 --- a/.github/actions/setup-binary-builds/action.yml +++ b/.github/actions/setup-binary-builds/action.yml @@ -40,6 +40,11 @@ inputs: required: false type: boolean default: no + is-manylinux2-28: + description: One of the parameter used by pkg-helpers + required: false + type: boolean + default: true runs: using: composite @@ -48,15 +53,26 @@ runs: shell: bash env: REPOSITORY: ${{ inputs.repository }} + IS_MANYLINUX2_28: ${{ inputs.is-manylinux2-28 }} run: | set -euxo pipefail rm -rf "${REPOSITORY}" + - uses: actions/checkout@v4 + if: ${{ env.IS_MANYLINUX2_28 == 'true' }} with: repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} submodules: ${{ inputs.submodules }} path: ${{ inputs.repository }} + + - uses: atalman/checkout-action@main + if: ${{ env.IS_MANYLINUX2_28 != 'true' }} + with: + repository: ${{ inputs.repository }} + path: ${{ inputs.repository }} + ref: ${{ inputs.ref }} + - name: Log Available Webhook Fields shell: bash run: | diff --git a/.github/workflows/_binary_upload.yml b/.github/workflows/_binary_upload.yml index ece5991aa8..604258c122 100644 --- a/.github/workflows/_binary_upload.yml +++ b/.github/workflows/_binary_upload.yml @@ -50,6 +50,8 @@ jobs: matrix: ${{ fromJSON(inputs.build-matrix) }} timeout-minutes: 30 name: ${{ matrix.build_name }} + env: + SKIP-UPLOAD: ${{ !contains(matrix.container_image, '2_28') }} steps: - uses: actions/checkout@v4 with: @@ -70,6 +72,7 @@ jobs: upload-to-base-bucket: ${{ matrix.upload_to_base_bucket }} - name: Download the artifact + if: ${{ !env.SKIP-UPLOAD }} uses: actions/download-artifact@v3 with: name: ${{ env.ARTIFACT_NAME }} @@ -97,6 +100,7 @@ jobs: echo "NIGHTLY_OR_TEST=1" >> "${GITHUB_ENV}" - name: Upload package to pytorch.org + if: ${{ !env.SKIP-UPLOAD }} shell: bash working-directory: ${{ inputs.repository }} run: | @@ -117,7 +121,7 @@ jobs: done - name: Upload package to pypi - if: ${{ env.NIGHTLY_OR_TEST == '1' && contains(inputs.upload-to-pypi, matrix.desired_cuda) }} + if: ${{ !env.SKIP-UPLOAD && env.NIGHTLY_OR_TEST == '1' && contains(inputs.upload-to-pypi, matrix.desired_cuda) }} uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml index 2813ef1e19..01e15dd424 100644 --- a/.github/workflows/build_wheels_linux.yml +++ b/.github/workflows/build_wheels_linux.yml @@ -123,6 +123,7 @@ jobs: CU_VERSION: ${{ matrix.desired_cuda }} UPLOAD_TO_BASE_BUCKET: ${{ matrix.upload_to_base_bucket }} ARCH: ${{ inputs.architecture }} + IS_MANYLINUX2_28: ${{ contains(matrix.container_image, '2_28') }} name: ${{ matrix.build_name }} runs-on: ${{ matrix.validation_runner }} container: @@ -141,15 +142,24 @@ jobs: if [[ "${{ inputs.architecture }}" = "aarch64" ]]; then rm -rf "${RUNNER_TEMP}/*" fi - echo "::endgroup::" - - uses: actions/checkout@v4 + + - uses: actions/checkout@v3 + if: ${{ env.IS_MANYLINUX2_28 == 'true' }} with: # Support the use case where we need to checkout someone's fork repository: ${{ inputs.test-infra-repository }} ref: ${{ inputs.test-infra-ref }} path: test-infra - - name: Set linux aarch64 CI + + - uses: atalman/checkout-action@main + if: ${{ env.IS_MANYLINUX2_28 != 'true' }} + with: + repository: ${{ inputs.test-infra-repository }} + ref: ${{ inputs.test-infra-ref }} + path: test-infra + + - name: Install Miniforge if: ${{ inputs.architecture == 'aarch64' }} shell: bash -l {0} env: @@ -157,7 +167,6 @@ jobs: run: | set -euxo pipefail # TODO: Get rid of Conda, we already have all versions of PyThon one needs in the docker - ############################################################################### # Install conda # disable SSL_verify due to getting "Could not find a suitable TLS CA certificate bundle, invalid path" @@ -185,10 +194,12 @@ jobs: repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} submodules: ${{ inputs.submodules }} - setup-miniconda: ${{ inputs.setup-miniconda }} + setup-miniconda: false python-version: ${{ env.PYTHON_VERSION }} cuda-version: ${{ env.CU_VERSION }} arch: ${{ env.ARCH }} + is-manylinux2-28: ${{ env.IS_MANYLINUX2_28 }} + - name: Combine Env Var and Build Env Files if: ${{ inputs.env-var-script != '' }} working-directory: ${{ inputs.repository }} @@ -281,7 +292,16 @@ jobs: ${CONDA_RUN} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}" fi # NB: Only upload to GitHub after passing smoke tests + + - name: Upload wheel + if: ${{ env.IS_MANYLINUX2_28 != 'true' }} + uses: ./test-infra/.github/actions/binary-upload + with: + repository: ${{ inputs.repository }} + trigger-event: ${{ inputs.trigger-event }} + - name: Upload wheel to GitHub + if: ${{ env.IS_MANYLINUX2_28 == 'true' }} continue-on-error: true uses: actions/upload-artifact@v3 with: