Try alternative to checkout

test test test test test test test test fix test test test test test test test test test test test test test test test test test test test test test test test test test test test fix test
pytorch · Nov 20, 2024 · 4e360ab · 4e360ab
1 parent a4e97ec
commit 4e360ab
Show file tree

Hide file tree

Showing 4 changed files with 103 additions and 6 deletions.
diff --git a/.github/actions/binary-upload/action.yml b/.github/actions/binary-upload/action.yml
@@ -0,0 +1,57 @@
+name: Binary upload action
+
+description: pull a specific docker image
+
+inputs:
+  repository:
+    description: 'Repository to checkout, defaults to ""'
+    default: ''
+    type: string
+  trigger-event:
+    description: "Trigger Event in caller that determines whether or not to upload"
+    type: string
+    default: ''
+
+runs:
+  using: composite
+  steps:
+    - name: Configure aws credentials (pytorch account)
+      if: ${{ inputs.trigger-event == 'schedule' || (inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly')) }}
+      uses: aws-actions/configure-aws-credentials@v3
+      with:
+        role-to-assume: arn:aws:iam::749337293305:role/gha_workflow_nightly_build_wheels
+        aws-region: us-east-1
+
+    - name: Configure aws credentials (pytorch account)
+      if: ${{ env.CHANNEL == 'test' && startsWith(github.event.ref, 'refs/tags/v') }}
+      uses: aws-actions/configure-aws-credentials@v3
+      with:
+        role-to-assume: arn:aws:iam::749337293305:role/gha_workflow_test_build_wheels
+        aws-region: us-east-1
+
+    - name: Nightly or release RC
+      if: ${{ inputs.trigger-event == 'schedule' || (inputs.trigger-event == 'push' && startsWith(github.event.ref, 'refs/heads/nightly')) || (env.CHANNEL == 'test' && startsWith(github.event.ref, 'refs/tags/')) }}
+      shell: bash
+      run: |
+        set -ex
+        echo "NIGHTLY_OR_TEST=1" >> "${GITHUB_ENV}"
+
+    - name: Upload package to pytorch.org
+      shell: bash
+      working-directory: ${{ inputs.repository }}
+      run: |
+        set -ex
+
+        # shellcheck disable=SC1090
+        source "${BUILD_ENV_FILE}"
+
+        pip install awscli==1.32.18
+
+        AWS_CMD="aws s3 cp --dryrun"
+        if [[ "${NIGHTLY_OR_TEST:-0}" == "1" ]]; then
+          AWS_CMD="aws s3 cp"
+        fi
+
+        for pkg in dist/*; do
+          ${AWS_CMD} "$pkg" "${PYTORCH_S3_BUCKET_PATH}" --acl public-read
+        done
diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml
@@ -40,6 +40,11 @@ inputs:
     required: false
     type: boolean
     default: no
+  is-manylinux2-28:
+    description: One of the parameter used by pkg-helpers
+    required: false
+    type: boolean
+    default: true
 
 runs:
   using: composite
@@ -48,15 +53,26 @@ runs:
         shell: bash
         env:
           REPOSITORY: ${{ inputs.repository }}
+          IS_MANYLINUX2_28: ${{ inputs.is-manylinux2-28 }}
         run: |
           set -euxo pipefail
           rm -rf "${REPOSITORY}"
+
       - uses: actions/checkout@v4
+        if: ${{ env.IS_MANYLINUX2_28 == 'true' }}
         with:
           repository: ${{ inputs.repository }}
           ref: ${{ inputs.ref }}
           submodules: ${{ inputs.submodules }}
           path: ${{ inputs.repository }}
+
+      - uses: atalman/checkout-action@main
+        if:  ${{ env.IS_MANYLINUX2_28 != 'true' }}
+        with:
+          repository: ${{ inputs.repository }}
+          path: ${{ inputs.repository }}
+          ref: ${{ inputs.ref }}
+
       - name: Log Available Webhook Fields
         shell: bash
         run: |

diff --git a/.github/workflows/_binary_upload.yml b/.github/workflows/_binary_upload.yml
@@ -50,6 +50,8 @@ jobs:
       matrix: ${{ fromJSON(inputs.build-matrix) }}
     timeout-minutes: 30
     name: ${{ matrix.build_name }}
+    env:
+      SKIP-UPLOAD: ${{ !contains(matrix.container_image, '2_28') }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -70,6 +72,7 @@ jobs:
           upload-to-base-bucket: ${{ matrix.upload_to_base_bucket }}
 
       - name: Download the artifact
+        if: ${{ !env.SKIP-UPLOAD }}
         uses: actions/download-artifact@v3
         with:
           name: ${{ env.ARTIFACT_NAME }}
@@ -97,6 +100,7 @@ jobs:
           echo "NIGHTLY_OR_TEST=1" >> "${GITHUB_ENV}"
 
       - name: Upload package to pytorch.org
+        if: ${{ !env.SKIP-UPLOAD }}
         shell: bash
         working-directory: ${{ inputs.repository }}
         run: |
@@ -117,7 +121,7 @@ jobs:
           done
 
       - name: Upload package to pypi
-        if: ${{ env.NIGHTLY_OR_TEST == '1' && contains(inputs.upload-to-pypi, matrix.desired_cuda) }}
+        if: ${{ !env.SKIP-UPLOAD  && env.NIGHTLY_OR_TEST == '1' && contains(inputs.upload-to-pypi, matrix.desired_cuda) }}
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           user: __token__

diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml
@@ -123,6 +123,7 @@ jobs:
       CU_VERSION: ${{ matrix.desired_cuda }}
       UPLOAD_TO_BASE_BUCKET: ${{ matrix.upload_to_base_bucket }}
       ARCH: ${{ inputs.architecture }}
+      IS_MANYLINUX2_28: ${{ contains(matrix.container_image, '2_28') }}
     name: ${{ matrix.build_name }}
     runs-on: ${{ matrix.validation_runner }}
     container:
@@ -141,23 +142,31 @@ jobs:
           if [[ "${{ inputs.architecture }}" = "aarch64" ]]; then
             rm -rf "${RUNNER_TEMP}/*"
           fi
-
           echo "::endgroup::"
-      - uses: actions/checkout@v4
+
+      - uses: actions/checkout@v3
+        if: ${{ env.IS_MANYLINUX2_28 == 'true' }}
         with:
           # Support the use case where we need to checkout someone's fork
           repository: ${{ inputs.test-infra-repository }}
           ref: ${{ inputs.test-infra-ref }}
           path: test-infra
-      - name: Set linux aarch64 CI
+
+      - uses: atalman/checkout-action@main
+        if:  ${{ env.IS_MANYLINUX2_28 != 'true' }}
+        with:
+          repository: ${{ inputs.test-infra-repository }}
+          ref: ${{ inputs.test-infra-ref }}
+          path: test-infra
+
+      - name: Install Miniforge
         if: ${{ inputs.architecture == 'aarch64' }}
         shell: bash -l {0}
         env:
           DESIRED_PYTHON: ${{ matrix.python_version }}
         run: |
           set -euxo pipefail
           # TODO: Get rid of Conda, we already have all versions of PyThon one needs in the docker
-
           ###############################################################################
           # Install conda
           # disable SSL_verify due to getting "Could not find a suitable TLS CA certificate bundle, invalid path"
@@ -185,10 +194,12 @@ jobs:
           repository: ${{ inputs.repository }}
           ref: ${{ inputs.ref }}
           submodules: ${{ inputs.submodules }}
-          setup-miniconda: ${{ inputs.setup-miniconda }}
+          setup-miniconda: false
           python-version: ${{ env.PYTHON_VERSION }}
           cuda-version: ${{ env.CU_VERSION }}
           arch: ${{ env.ARCH }}
+          is-manylinux2-28: ${{ env.IS_MANYLINUX2_28 }}
+
       - name: Combine Env Var and Build Env Files
         if: ${{ inputs.env-var-script != '' }}
         working-directory: ${{ inputs.repository }}
@@ -281,7 +292,16 @@ jobs:
             ${CONDA_RUN} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}"
           fi
       # NB: Only upload to GitHub after passing smoke tests
+
+      - name: Upload wheel
+        if: ${{ env.IS_MANYLINUX2_28 != 'true' }}
+        uses: ./test-infra/.github/actions/binary-upload
+        with:
+          repository: ${{ inputs.repository }}
+          trigger-event: ${{ inputs.trigger-event }}
+
       - name: Upload wheel to GitHub
+        if: ${{ env.IS_MANYLINUX2_28 == 'true' }}
         continue-on-error: true
         uses: actions/upload-artifact@v3
         with: