build_release_candidate.yml

name: build_release_candidate

# Workflow added after https://github.com/apache/beam/commit/4183e747becebd18becee5fff547af365910fc9c
# If help is needed debugging issues, you can view the release guide at that commit for guidance on how to do this manually.
# (https://github.com/apache/beam/blob/4183e747becebd18becee5fff547af365910fc9c/website/www/site/content/en/contribute/release-guide.md)
on:
  workflow_dispatch:
    inputs:
      RELEASE:
        description: Beam version of current release (e.g. 2.XX.0)
        required: true
        default: '2.XX.0'
      RC:
        description: Integer RC version for the release (e.g. 3 for RC3)
        required: true
      APACHE_ID:
        description: Your Apache id. Required if you want to stage artifacts into https://dist.apache.org/repos/dist/dev/beam/
        required: false
      APACHE_PASSWORD:
        description: Your Apache password. Required if you want to stage artifacts into https://dist.apache.org/repos/dist/dev/beam/
        required: false
      BEAM_SITE_TOKEN:
        description:  Github Personal Access Token with apache/beam-site repo permission if you want to create the beam-site docs PR. See https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens.
        default: ''
      PUBLISH_JAVA_ARTIFACTS:
        description: Whether to publish java artifacts to https://repository.apache.org/#stagingRepositories (yes/no)
        required: true
        default: 'no'
      STAGE_JAVA_SOURCE:
        description: Whether to stage the java source into https://dist.apache.org/repos/dist/dev/beam/
        required: true
        default: 'no'
      STAGE_DOCKER_ARTIFACTS:
        description: Whether to stage SDK docker images to docker hub Apache organization
        required: true
        default: 'no'
      STAGE_PYTHON_ARTIFACTS:
        description: Whether to stage the python artifacts into https://dist.apache.org/repos/dist/dev/beam/
        required: true
        default: 'no'
      CREATE_BEAM_SITE_PR:
        description: Whether to create the documentation update PR against apache/beam-site.
        required: true
        default: 'no'

env:
  GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
  GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
  GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}

jobs:
  publish_java_artifacts:
    if: ${{github.event.inputs.PUBLISH_JAVA_ARTIFACTS == 'yes'}}
    runs-on: [self-hosted, ubuntu-20.04, main]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
          repository: apache/beam
      - name: Install Java 8
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: '8'
      - name: Import GPG key
        id: import_gpg
        uses: crazy-max/ghaction-import-gpg@111c56156bcc6918c056dbef52164cfa583dc549
        with:
          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
      - name: Auth for nexus
        run: |
          mkdir -p ${HOME}/.m2
          echo "<settings>
           <servers>
             <server>
               <id>apache.releases.https</id>
               <username>${{ secrets.NEXUS_STAGE_DEPLOYER_USER }}</username>
               <password>${{ secrets.NEXUS_STAGE_DEPLOYER_PW }}</password>
             </server>
             <server>
               <id>apache.snapshots.https</id>
               <username>${{ secrets.NEXUS_USER }}</username>
               <password>${{ secrets.NEXUS_PW }}</password>
             </server>
           </servers>
          </settings>" > ${HOME}/.m2/settings.xml
      - name: Configure git
        run: git config credential.helper store
      - name: Stage Java Artifacts into Maven
        run: ./gradlew publish -Psigning.gnupg.keyName=${{steps.import_gpg.outputs.fingerprint}} -PisRelease --no-daemon --no-parallel
        
      
  stage_java_source:
    if: ${{github.event.inputs.STAGE_JAVA_SOURCE == 'yes'}}
    runs-on: ubuntu-latest
    steps:
      - name: Mask Apache Password
        run: |
          # Workaround for Actions bug - https://github.com/actions/runner/issues/643
          APACHE_PASS=$(jq -r '.inputs.APACHE_PASSWORD' $GITHUB_EVENT_PATH)
          echo "::add-mask::$APACHE_PASS"
      - name: Validate apache id/password
        run: |
          if [ "${{ github.event.inputs.APACHE_ID }}" == "" ]
          then
            echo "Must provide an apache id to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/"
          fi
          if [ "${{ github.event.inputs.APACHE_PASSWORD }}" == "" ]
          then
            echo "Must provide an apache password to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/"
          fi
      - name: Install Java 8
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: '8'
      - name: Import GPG key
        id: import_gpg
        uses: crazy-max/ghaction-import-gpg@111c56156bcc6918c056dbef52164cfa583dc549
        with:
          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
      - name: stage source
        run: |
            svn co https://dist.apache.org/repos/dist/dev/beam
            mkdir -p beam/${{ github.event.inputs.RELEASE }}
            cd beam/${{ github.event.inputs.RELEASE }}
            RC_DIR="beam-${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
            RC_ZIP="${RC_DIR}.zip"
            RELEASE_DIR="beam-${{ github.event.inputs.RELEASE }}"
            RC_TAG="v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
            SOURCE_RELEASE_ZIP="apache-beam-${{ github.event.inputs.RELEASE }}-source-release.zip"
              # Check whether there is an existing dist dir
            if (svn ls "${SOURCE_RELEASE_ZIP}"); then
              echo "Removing existing ${SOURCE_RELEASE_ZIP}."
              svn delete "${SOURCE_RELEASE_ZIP}"
            fi

            echo "Downloading: https://github.com/apache/beam/archive/${RC_TAG}.zip"
            wget https://github.com/apache/beam/archive/${RC_TAG}.zip  -O "${RC_ZIP}"

            unzip "$RC_ZIP"
            rm "$RC_ZIP"
            mv "$RC_DIR" "$RELEASE_DIR"
            zip -r "${SOURCE_RELEASE_ZIP}" "$RELEASE_DIR"
            rm -r "$RELEASE_DIR"

            echo "----Signing Source Release ${SOURCE_RELEASE_ZIP}-----"
            gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --batch --yes --detach-sig "${SOURCE_RELEASE_ZIP}"

            echo "----Creating Hash Value for ${SOURCE_RELEASE_ZIP}----"
            sha512sum ${SOURCE_RELEASE_ZIP} > ${SOURCE_RELEASE_ZIP}.sha512

            svn add --force .
            svn status
            svn commit -m "Staging Java artifacts for Apache Beam ${{ github.event.inputs.RELEASE }} RC${{ github.event.inputs.RC }}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}" 
        
  stage_python_artifacts:
    if: ${{github.event.inputs.STAGE_PYTHON_ARTIFACTS == 'yes'}}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Mask Apache Password
        run: |
          # Workaround for Actions bug - https://github.com/actions/runner/issues/643
          APACHE_PASS=$(jq -r '.inputs.APACHE_PASSWORD' $GITHUB_EVENT_PATH)
          echo "::add-mask::$APACHE_PASS"
      - name: Mask apache id/password
        run: |
          if [ "${{ github.event.inputs.APACHE_ID }}" == "" ]
          then
            echo "Must provide an apache id to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/"
          fi
          if [ "${{ github.event.inputs.APACHE_PASSWORD }}" == "" ]
          then
            echo "Must provide an apache password to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/"
          fi
      - name: Setup environment
        uses: ./.github/actions/setup-environment-action
        with:
          python-version: 3.8
      - name: Import GPG key
        id: import_gpg
        uses: crazy-max/ghaction-import-gpg@111c56156bcc6918c056dbef52164cfa583dc549
        with:
          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
      - name: Install dependencies
        run: |
          pip install python-dateutil
          pip install requests
      - name: stage python artifacts
        env:
          RC_TAG: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
          GIT_REPO_BASE_URL: https://github.com/apache/beam
          RELEASE_DIR: "beam/${{ github.event.inputs.RELEASE }}"
          RELEASE: "${{ github.event.inputs.RELEASE }}"
          SCRIPT_DIR: release/src/main/scripts
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SVN_ARTIFACTS_DIR: "beam/${{ github.event.inputs.RELEASE }}/python"
        run: |
          svn co https://dist.apache.org/repos/dist/dev/beam
          mkdir -p "${SVN_ARTIFACTS_DIR}"

          git fetch --all --tags --prune
          RELEASE_COMMIT=$(git rev-list -n 1 "tags/${RC_TAG}")
          
          python "${SCRIPT_DIR}/download_github_actions_artifacts.py" \
            --github-token-var GITHUB_TOKEN \
            --repo-url "apache/beam" \
            --rc-tag "${RC_TAG}" \
            --release-commit "${RELEASE_COMMIT}" \
            --artifacts_dir "${RELEASE_DIR}/python" \
            --yes True
        
          cd "${RELEASE_DIR}"/python
          ls

          echo "------Checking Hash Value for apache-beam-${RELEASE}.tar.gz-----"
          sha512sum -c "apache-beam-${RELEASE}.tar.gz.sha512"
        
          echo "------Signing Source Release apache-beam-${RELEASE}.tar.gz------"
          gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --detach-sig "apache-beam-${RELEASE}.tar.gz"
        
          for artifact in *.whl; do
            echo "----------Checking Hash Value for ${artifact} wheel-----------"
            sha512sum -c "${artifact}.sha512"
          done
        
          for artifact in *.whl; do
            echo "------------------Signing ${artifact} wheel-------------------"
            gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --batch --yes --detach-sig "${artifact}"
          done
        
          cd ..
          svn add --force python
          svn status
          svn commit -m "Staging Python artifacts for Apache Beam ${RELEASE} RC${RC_NUM}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}"
        

  stage_docker:
    if: ${{github.event.inputs.STAGE_DOCKER_ARTIFACTS == 'yes'}}
    # Note: if this ever changes to self-hosted, remove the "Remove default github maven configuration" step
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
          repository: apache/beam
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@v1.3.0
      - name: Install Java 11
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: '11'
      - name: Install Python 3.8
        uses: actions/setup-python@v5
        with:
          python-version: '3.8'
      - run: echo $JAVA_HOME
      - run: echo "JAVA11_HOME=${JAVA_HOME}" >> "$GITHUB_OUTPUT"
        id: export-java11
      - name: Install Java 8
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: '8'
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      - name: Remove default github maven configuration
        # This step is a workaround to avoid a decryption issue of Beam's
        # net.linguica.gradle.maven.settings plugin and github's provided maven
        # settings.xml file
        run: rm ~/.m2/settings.xml || true
      - name: Login to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USER }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Push docker images
        run: ./gradlew :pushAllDockerImages -PisRelease -Pdocker-pull-licenses -Pprune-images -Pdocker-tag=${{ github.event.inputs.RELEASE }}rc${{ github.event.inputs.RC }} -Pjava11Home=${{steps.export-java11.outputs.JAVA11_HOME}} --no-daemon --no-parallel

  beam_site_pr:
    if: ${{github.event.inputs.CREATE_BEAM_SITE_PR == 'yes'}}
    # Note: if this ever changes to self-hosted, remove the "Remove default github maven configuration" step
    runs-on: ubuntu-latest
    env:
      RC_TAG: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
      BRANCH_NAME: updates_release_${{ github.event.inputs.RELEASE }}
      BEAM_ROOT_DIR: ${{ github.workspace }}/beam
      SITE_ROOT_DIR: ${{ github.workspace }}/beam-site
    steps:
      - name: Checkout Beam Repo
        uses: actions/checkout@v4
        with:
          ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
          repository: apache/beam
          path: beam
      - name: Checkout Beam Site Repo
        uses: actions/checkout@v4
        with:
          repository: apache/beam-site
          path: beam-site
          token: ${{ github.event.inputs.BEAM_SITE_TOKEN }}
          ref: release-docs
      - name: Install Python 3.8
        uses: actions/setup-python@v5
        with:
          python-version: '3.8'
      - name: Install node
        uses: actions/setup-node@v4
        with:
          node-version: '16'
      - name: Install Java 8
        uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: '8'
      - name: Remove default github maven configuration
        # This step is a workaround to avoid a decryption issue of Beam's
        # net.linguica.gradle.maven.settings plugin and github's provided maven
        # settings.xml file
        run: rm ~/.m2/settings.xml || true
      - name: Create documentation commit for website
        run: |
            echo "OK!"
      - name: Build Python Docs
        working-directory: beam/sdks/python
        run: |
            pip install --upgrade pip setuptools wheel
            pip install -U pip
            pip install tox
            # TODO(https://github.com/apache/beam/issues/20209): Don't hardcode py version in this file.
            tox -e py38-docs
            rm -rf target/docs/_build/.doctrees
      - name: Build Typescript Docs
        working-directory: beam/sdks/typescript
        run: |
            npm ci && npm run docs
      - name: Build Java Docs
        working-directory: beam
        run: |
            ./gradlew :sdks:java:javadoc:aggregateJavadoc -PisRelease --no-daemon --no-parallel
      - name: Add canonical link into javadocs
        uses: cicirello/javadoc-cleanup@v1
        with:
          path-to-root: beam/sdks/java/javadoc/build/docs/javadoc
          base-url-path: https://beam.apache.org/releases/javadoc/current/
      - name: Consolidate Release Docs to beam-site branch with symlinks
        working-directory: beam-site
        run: |
            git checkout -b $BRANCH_NAME release-docs
             
            echo "..........Copying generated javadoc into beam-site.........."
            cp -r ${BEAM_ROOT_DIR}/sdks/java/javadoc/build/docs/javadoc/ javadoc/${{ github.event.inputs.RELEASE }}
            # Update current symlink to point to the latest release
            unlink javadoc/current
            ln -s ${{ github.event.inputs.RELEASE }} javadoc/current
             
            echo "............Copying generated pydoc into beam-site.........."
            cp -r ${BEAM_ROOT_DIR}/sdks/python/target/docs/_build pydoc/${{  github.event.inputs.RELEASE }}
            # Update current symlink to point to the latest release
            unlink pydoc/current
            ln -s ${{ github.event.inputs.RELEASE }} pydoc/current
             
            echo "............Copying generated typedoc into beam-site.........."
            mkdir -p typedoc
            cp -r ${BEAM_ROOT_DIR}/sdks/typescript/docs typedoc/${{ github.event.inputs.RELEASE }}
            # Update current symlink to point to the latest release
            unlink typedoc/current | true
            ln -s ${{ github.event.inputs.RELEASE }} typedoc/current
      - name: Create commit on beam-site branch
        working-directory: beam
        run: |
            # Get the commit from the beam repo, not the beam-site repo.
            RELEASE_COMMIT=$(git rev-list -n 1 "tags/${RC_TAG}")
            cd ${SITE_ROOT_DIR}
            git config user.name $GITHUB_ACTOR
            git config user.email actions@"$RUNNER_NAME".local
            git add -A
            git commit -m "Update beam-site for release ${{ github.event.inputs.RELEASE }}." -m "Content generated from commit ${RELEASE_COMMIT}."
            git push -f --set-upstream origin $BRANCH_NAME
      - name: Create beam-site PR
        working-directory: beam-site
        env:
          GH_TOKEN: ${{ github.event.inputs.BEAM_SITE_TOKEN }}
          PR_TITLE: "Publish docs for ${{ github.event.inputs.RELEASE }} release"
          PR_BODY: "Content generated from https://github.com/apache/beam/tree/${{ env.RC_TAG }}."
        run: |
            gh pr create -t "$PR_TITLE" -b "$PR_BODY" --base release-docs