change extension of beam sdist from .zip to .tar.gz (#29108)

apache · Oct 27, 2023 · e8c455e · e8c455e
1 parent 6bb3ef2
commit e8c455e
Show file tree

Hide file tree

Showing 9 changed files with 104 additions and 105 deletions.
diff --git a/.github/actions/common-rc-validation/action.yaml b/.github/actions/common-rc-validation/action.yaml
@@ -36,19 +36,19 @@ runs:
       shell: bash
       run: |
           echo "---------------------Downloading Python Staging RC----------------------------"
-          wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip
-          wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512
-          if [[ ! -f apache-beam-$RELEASE_VER.zip ]]; then
+          wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.tar.gz
+          wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.tar.gz.sha512
+          if [[ ! -f apache-beam-$RELEASE_VER.tar.gz ]]; then
             { echo "Fail to download Python Staging RC files." ;exit 1; }
           fi
 
           echo "--------------------------Verifying Hashes------------------------------------"
-          sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
+          sha512sum -c apache-beam-${RELEASE_VER}.tar.gz.sha512
 
           `which pip` install --upgrade pip
           `which pip` install --upgrade setuptools
 
     - name: Installing python SDK
       shell: bash
-      run: pip install apache-beam-${RELEASE_VER}.zip[gcp]
+      run: pip install apache-beam-${RELEASE_VER}.tar.gz[gcp]
 
diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml
@@ -122,18 +122,18 @@ jobs:
             mkdir -p beam/${{ github.event.inputs.RELEASE }}
             cd beam/${{ github.event.inputs.RELEASE }}
             RC_DIR="beam-${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
-            RC_ZIP="${RC_DIR}.zip"
+            RC_ZIP="${RC_DIR}.tar.gz"
             RELEASE_DIR="beam-${{ github.event.inputs.RELEASE }}"
             RC_TAG="v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
-            SOURCE_RELEASE_ZIP="apache-beam-${{ github.event.inputs.RELEASE }}-source-release.zip"
+            SOURCE_RELEASE_ZIP="apache-beam-${{ github.event.inputs.RELEASE }}-source-release.tar.gz"
               # Check whether there is an existing dist dir
             if (svn ls "${SOURCE_RELEASE_ZIP}"); then
               echo "Removing existing ${SOURCE_RELEASE_ZIP}."
               svn delete "${SOURCE_RELEASE_ZIP}"
             fi
 
-            echo "Downloading: https://github.com/apache/beam/archive/${RC_TAG}.zip"
-            wget https://github.com/apache/beam/archive/${RC_TAG}.zip  -O "${RC_ZIP}"
+            echo "Downloading: https://github.com/apache/beam/archive/${RC_TAG}.tar.gz"
+            wget https://github.com/apache/beam/archive/${RC_TAG}.tar.gz  -O "${RC_ZIP}"
 
             unzip "$RC_ZIP"
             rm "$RC_ZIP"
@@ -197,11 +197,11 @@ jobs:
         
           cd "${RELEASE_DIR}"/python
         
-          echo "------Checking Hash Value for apache-beam-${RELEASE}.zip-----"
-          sha512sum -c "apache-beam-${RELEASE}.zip.sha512"
+          echo "------Checking Hash Value for apache-beam-${RELEASE}.tar.gz-----"
+          sha512sum -c "apache-beam-${RELEASE}.tar.gz.sha512"
         
-          echo "------Signing Source Release apache-beam-${RELEASE}.zip------"
-          gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --detach-sig "apache-beam-${RELEASE}.zip"
+          echo "------Signing Source Release apache-beam-${RELEASE}.tar.gz------"
+          gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --detach-sig "apache-beam-${RELEASE}.tar.gz"
         
           for artifact in *.whl; do
             echo "----------Checking Hash Value for ${artifact} wheel-----------"

diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml
@@ -17,14 +17,14 @@
 
 # To learn more about GitHub Actions in Apache Beam check the CI.m
 
-name: Run RC Validation 
+name: Run RC Validation
 on:
   workflow_dispatch:
     inputs:
       RELEASE_VER:
         description: Beam current Release Version
         required: true
-        default: 2.42.0 
+        default: 2.42.0
       USER_GCS_BUCKET:
         description: Bucket to upload results
         required: true
@@ -57,9 +57,9 @@ on:
         type: boolean
         required: true
         default: true
-env: 
+env:
   RC_TAG: "v${{github.event.inputs.RELEASE_VER}}-RC${{github.event.inputs.RC_NUM}}"
-  RELEASE_VER: ${{github.event.inputs.RELEASE_VER}}    
+  RELEASE_VER: ${{github.event.inputs.RELEASE_VER}}
   USER_GCP_PROJECT: apache-beam-testing
   PYTHON_RC_DOWNLOAD_URL: https://dist.apache.org/repos/dist/dev/beam
   USER_GCP_REGION: us-central1
@@ -100,19 +100,19 @@ jobs:
       - name: Comment on PR to Trigger Python ReleaseCandidate Test
         run: |
           gh pr comment "$GITHUB_PR_URL" --body "Run Python ReleaseCandidate"
-  
+
   sql_taxi_with_dataflow:
     runs-on: [self-hosted,ubuntu-20.04]
     if: ${{github.event.inputs.RUN_SQL_TAXI_WITH_DATAFLOW == 'true'}}
     strategy:
-      matrix: 
+      matrix:
         py_version: [3.8]
     steps:
     - name: Checkout code
       uses: actions/checkout@v4
-      with: 
-       ref: ${{env.RC_TAG}}      
-    
+      with:
+       ref: ${{env.RC_TAG}}
+
     - name: Install Python
       uses: actions/setup-python@v4
       with:
@@ -154,23 +154,23 @@ jobs:
         --num_workers 5 \
         --output_topic projects/${USER_GCP_PROJECT}/topics/${SQL_TAXI_TOPIC} \
         --beam_services="{\":sdks:java:extensions:sql:expansion-service:shadowJar\": \"${SQL_EXPANSION_SERVICE_JAR}\"}" \
-        --sdk_location apache-beam-${RELEASE_VER}.zip || true
+        --sdk_location apache-beam-${RELEASE_VER}.tar.gz || true
     - name: Checking Results
       run: |
         gcloud pubsub subscriptions pull --project=${USER_GCP_PROJECT} --limit=5 ${SQL_TAXI_SUBSCRIPTION}
         gcloud pubsub subscriptions pull --project=${USER_GCP_PROJECT} --limit=5 ${SQL_TAXI_SUBSCRIPTION}
     - name: Removing Pub Sub Topic
       if: always()
-      run: | 
+      run: |
         gcloud pubsub topics delete --project=${USER_GCP_PROJECT} ${SQL_TAXI_TOPIC}
         gcloud pubsub subscriptions delete --project=${USER_GCP_PROJECT} ${SQL_TAXI_SUBSCRIPTION}
 
 
   python_cross_validation:
     runs-on: [self-hosted,ubuntu-20.04]
-    if: ${{github.event.inputs.RUN_PYTHON_CROSS_VALIDATION == 'true'}} 
+    if: ${{github.event.inputs.RUN_PYTHON_CROSS_VALIDATION == 'true'}}
     strategy:
-      matrix: 
+      matrix:
         py_version: [3.8]
     steps:
     - name: Checkout code
@@ -183,7 +183,7 @@ jobs:
         echo "====================Checking Environment & Variables================="
         echo ""
         echo "running validations on release ${{github.event.inputs.RELEASE_VER}} RC${{github.event.inputs.RC_NUM}}."
-    - name: Install Kubectl 
+    - name: Install Kubectl
       uses: azure/setup-kubectl@v3
 
     - name: Setup Java JDK
@@ -196,7 +196,7 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: ${{matrix.py_version}}
-   
+
 
     - name: Setting python env
       uses: ./.github/actions/common-rc-validation
@@ -206,19 +206,19 @@ jobs:
 
     - name: Installing gcloud-auth-plugin
       run: sudo apt-get install google-cloud-sdk-gke-gcloud-auth-plugin
-    - name: Setting Kafka Cluster Name 
+    - name: Setting Kafka Cluster Name
       run: |
-        echo "KAFKA_CLUSTER_NAME=xlang-kafka-cluster-$RANDOM">> $GITHUB_ENV 
-    
+        echo "KAFKA_CLUSTER_NAME=xlang-kafka-cluster-$RANDOM">> $GITHUB_ENV
+
 
     - name: Creating Kafka Cluster
       run: |
         gcloud container clusters create --project=${USER_GCP_PROJECT} --region=${USER_GCP_REGION} --no-enable-ip-alias $KAFKA_CLUSTER_NAME
         kubectl apply -R -f .test-infra/kubernetes/kafka-cluster
-    
+
     - name: Waiting for Kafka cluster to be ready
       run: kubectl wait --for=condition=Ready pod/kafka-0 --timeout=1200s
-    - name: Start xlang Kafka Taxi with Dataflow Runner 
+    - name: Start xlang Kafka Taxi with Dataflow Runner
       run: |
         echo "BOOTSTRAP_SERVERS=$(kubectl get svc outside-0 -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):32400" >> $GITHUB_ENV
         echo "KAFKA_TAXI_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
@@ -243,16 +243,16 @@ jobs:
         --temp_location=${USER_GCS_BUCKET}/temp/ \
         --with_metadata \
         --beam_services="{\"sdks:java:io:expansion-service:shadowJar\": \"${KAFKA_EXPANSION_SERVICE_JAR}\"}" \
-        --sdk_location apache-beam-${RELEASE_VER}.zip || true
+        --sdk_location apache-beam-${RELEASE_VER}.tar.gz || true
     - name: Checking executions results
       run: |
-        bq head -n 10 ${KAFKA_TAXI_DF_DATASET}.xlang_kafka_taxi          
+        bq head -n 10 ${KAFKA_TAXI_DF_DATASET}.xlang_kafka_taxi
     - name: Remove BigQuery Dataset
       if: always()
       run: |
         bq rm -f ${KAFKA_TAXI_DF_DATASET}.xlang_kafka_taxi
         bq rm -f ${KAFKA_TAXI_DF_DATASET}
-    
+
     - name: Delete Kafka Cluster
       if: always()
       run: gcloud container clusters delete --project=${USER_GCP_PROJECT} --region=${USER_GCP_REGION} --async -q $KAFKA_CLUSTER_NAME
@@ -266,8 +266,8 @@ jobs:
     steps:
     - name: Sending PubSub name to env
       run: |
-        echo "SHARED_PUBSUB_TOPIC=leader_board-${GITHUB_ACTOR}-python-topic-$(date +%m%d)_$RANDOM" >> $GITHUB_ENV 
-    - id: generate_pubsub_name 
+        echo "SHARED_PUBSUB_TOPIC=leader_board-${GITHUB_ACTOR}-python-topic-$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
+    - id: generate_pubsub_name
       run: |
         echo "::set-output name=pubsub::$SHARED_PUBSUB_TOPIC"
     - name: Creating Pub Sub Topics
@@ -287,7 +287,7 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: '3.8'
-    
+
     - name: Setting python env
       uses: ./.github/actions/common-rc-validation
       with:
@@ -343,14 +343,14 @@ jobs:
       run: |
         ls
         cd word-count-beam
-        timeout --preserve-status 50m mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.complete.game.injector.Injector -Dexec.args="${USER_GCP_PROJECT} ${{needs.generate_shared_pubsub.outputs.name}} none" || true 
-  
+        timeout --preserve-status 50m mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.complete.game.injector.Injector -Dexec.args="${USER_GCP_PROJECT} ${{needs.generate_shared_pubsub.outputs.name}} none" || true
+
 
   direct_runner_leaderboard:
     runs-on: [self-hosted, ubuntu-20.04]
     if: ${{github.event.inputs.RUN_DIRECT_RUNNER_TESTS == 'true' }}
     strategy:
-      matrix: 
+      matrix:
         py_version: [3.8]
     needs: generate_shared_pubsub
     steps:
@@ -363,13 +363,13 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{matrix.py_version}}
-      
+
       - name: Setting python env
         uses: ./.github/actions/common-rc-validation
         with:
           RELEASE_VER: ${{env.RELEASE_VER}}
           PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}}
-      
+
       - name: Exporting leaderboard Dataset Name
         run: echo "LEADERBOARD_DIRECT_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
       - name: Creating Dataset
@@ -389,11 +389,11 @@ jobs:
           bq head -n 10 ${LEADERBOARD_DIRECT_DATASET}.leader_board_teams
       - name: Removing BigQuery Dataset
         if: always()
-        run: | 
+        run: |
           bq rm -f ${LEADERBOARD_DIRECT_DATASET}.leader_board_users
           bq rm -f ${LEADERBOARD_DIRECT_DATASET}.leader_board_teams
           bq rm -f $LEADERBOARD_DIRECT_DATASET
-  
+
   dataflow_runner_leaderboard:
     runs-on: [self-hosted,ubuntu-20.04]
     if: ${{github.event.inputs.RUN_DATAFLOW_RUNNER_TESTS=='true'}}
@@ -411,13 +411,13 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{matrix.py_version}}
-      
+
       - name: Setting python env
         uses: ./.github/actions/common-rc-validation
         with:
           RELEASE_VER: ${{env.RELEASE_VER}}
           PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}}
-      
+
       - name: Exporting Dataflow Dataset Name
         run: echo "LEADERBOARD_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
       - name: Creating Dataset
@@ -434,18 +434,18 @@ jobs:
           --dataset ${LEADERBOARD_DF_DATASET} \
           --runner DataflowRunner \
           --temp_location=${USER_GCS_BUCKET}/temp/ \
-          --sdk_location apache-beam-${RELEASE_VER}.zip || true
+          --sdk_location apache-beam-${RELEASE_VER}.tar.gz || true
       - name: Checking results
         run: |
           bq head -n 10 ${LEADERBOARD_DF_DATASET}.leader_board_users
           bq head -n 10 ${LEADERBOARD_DF_DATASET}.leader_board_teams
       - name: Removing BigQuery Dataset
         if: always()
-        run: | 
+        run: |
           bq rm -f ${LEADERBOARD_DF_DATASET}.leader_board_users
           bq rm -f ${LEADERBOARD_DF_DATASET}.leader_board_teams
           bq rm -f $LEADERBOARD_DF_DATASET
-      
+
 
   direct_runner_gamestats:
     runs-on: [self-hosted,ubuntu-20.04]
@@ -463,13 +463,13 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{matrix.py_version}}
-   
+
       - name: Setting python env
         uses: ./.github/actions/common-rc-validation
         with:
           RELEASE_VER: ${{env.RELEASE_VER}}
           PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}}
-      
+
       - name: Exporting Gamestates Direct Dataset Name
         run: echo "GAMESTATS_DIRECT_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
       - name: Creating Dataset
@@ -490,7 +490,7 @@ jobs:
           bq head -n 10 ${GAMESTATS_DIRECT_DATASET}.game_stats_sessions
       - name: Removing BigQuery Dataset
         if: always()
-        run: | 
+        run: |
           bq rm -f ${GAMESTATS_DIRECT_DATASET}.game_stats_sessions
           bq rm -f ${GAMESTATS_DIRECT_DATASET}.game_stats_teams
           bq rm -f $GAMESTATS_DIRECT_DATASET
@@ -512,13 +512,13 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{matrix.py_version}}
-      
+
       - name: Setting python env
         uses: ./.github/actions/common-rc-validation
         with:
           RELEASE_VER: ${{env.RELEASE_VER}}
           PYTHON_RC_DOWNLOAD_URL: ${{env.PYTHON_RC_DOWNLOAD_URL}}
-      
+
       - name: Exporting Gamestates Direct Dataset Name
         run: echo "GAMESTATS_DF_DATASET=${GITHUB_ACTOR}_python_validations_$(date +%m%d)_$RANDOM" >> $GITHUB_ENV
       - name: Creating Dataset
@@ -535,7 +535,7 @@ jobs:
           --dataset ${GAMESTATS_DF_DATASET} \
           --runner DataflowRunner \
           --temp_location=${USER_GCS_BUCKET}/temp/ \
-          --sdk_location apache-beam-${RELEASE_VER}.zip \
+          --sdk_location apache-beam-${RELEASE_VER}.tar.gz \
           --fixed_window_duration ${FIXED_WINDOW_DURATION} || true
       - name: Checking Results
         run: |
@@ -545,14 +545,13 @@ jobs:
         if: always()
         run: |
           bq rm -f ${GAMESTATS_DF_DATASET}.game_stats_teams
-          bq rm -f ${GAMESTATS_DF_DATASET}.game_stats_sessions 
+          bq rm -f ${GAMESTATS_DF_DATASET}.game_stats_sessions
           bq rm -f $GAMESTATS_DF_DATASET
-  
+
   remove_shared_pubsub:
     runs-on: [self-hosted,ubuntu-20.04]
     needs: [java_injector, generate_shared_pubsub]
     if: always()
     steps:
       - name: Deleting Shared Pub Sub
         run: gcloud pubsub topics delete --project=${USER_GCP_PROJECT}  ${{needs.generate_shared_pubsub.outputs.name}}
-