diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml index 8d27cea1d5d5..71bddec922b4 100644 --- a/.github/workflows/build_release_candidate.yml +++ b/.github/workflows/build_release_candidate.yml @@ -89,8 +89,8 @@ jobs: run: git config credential.helper store - name: Stage Java Artifacts into Maven run: ./gradlew publish -Psigning.gnupg.keyName=${{steps.import_gpg.outputs.fingerprint}} -PisRelease --no-daemon --no-parallel - - + + stage_java_source: if: ${{github.event.inputs.STAGE_JAVA_SOURCE == 'yes'}} runs-on: ubuntu-latest @@ -153,8 +153,8 @@ jobs: svn add --force . svn status - svn commit -m "Staging Java artifacts for Apache Beam ${{ github.event.inputs.RELEASE }} RC${{ github.event.inputs.RC }}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}" - + svn commit -m "Staging Java artifacts for Apache Beam ${{ github.event.inputs.RELEASE }} RC${{ github.event.inputs.RC }}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}" + stage_python_artifacts: if: ${{github.event.inputs.STAGE_PYTHON_ARTIFACTS == 'yes'}} runs-on: ubuntu-latest @@ -204,7 +204,7 @@ jobs: git fetch --all --tags --prune RELEASE_COMMIT=$(git rev-list -n 1 "tags/${RC_TAG}") - + python "${SCRIPT_DIR}/download_github_actions_artifacts.py" \ --github-token-var GITHUB_TOKEN \ --repo-url "apache/beam" \ @@ -212,31 +212,31 @@ jobs: --release-commit "${RELEASE_COMMIT}" \ --artifacts_dir "${RELEASE_DIR}/python" \ --yes True - + cd "${RELEASE_DIR}"/python ls echo "------Checking Hash Value for apache-beam-${RELEASE}.tar.gz-----" sha512sum -c "apache-beam-${RELEASE}.tar.gz.sha512" - + echo "------Signing Source Release apache-beam-${RELEASE}.tar.gz------" gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --detach-sig "apache-beam-${RELEASE}.tar.gz" - + for artifact in *.whl; do echo "----------Checking Hash Value for ${artifact} wheel-----------" sha512sum -c "${artifact}.sha512" done - + for artifact in *.whl; do echo "------------------Signing ${artifact} wheel-------------------" gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --batch --yes --detach-sig "${artifact}" done - + cd .. svn add --force python svn status svn commit -m "Staging Python artifacts for Apache Beam ${RELEASE} RC${RC_NUM}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}" - + stage_docker: if: ${{github.event.inputs.STAGE_DOCKER_ARTIFACTS == 'yes'}} @@ -343,6 +343,10 @@ jobs: working-directory: beam run: | ./gradlew :sdks:java:javadoc:aggregateJavadoc -PisRelease --no-daemon --no-parallel + - name: Build YAML Docs + working-directory: beam + run: | + ./gradlew :sdks:python:generateYamlDocs -PisRelease - name: Add canonical link into javadocs uses: cicirello/javadoc-cleanup@v1 with: @@ -352,25 +356,32 @@ jobs: working-directory: beam-site run: | git checkout -b $BRANCH_NAME release-docs - + echo "..........Copying generated javadoc into beam-site.........." cp -r ${BEAM_ROOT_DIR}/sdks/java/javadoc/build/docs/javadoc/ javadoc/${{ github.event.inputs.RELEASE }} # Update current symlink to point to the latest release unlink javadoc/current ln -s ${{ github.event.inputs.RELEASE }} javadoc/current - + echo "............Copying generated pydoc into beam-site.........." cp -r ${BEAM_ROOT_DIR}/sdks/python/target/docs/_build pydoc/${{ github.event.inputs.RELEASE }} # Update current symlink to point to the latest release unlink pydoc/current ln -s ${{ github.event.inputs.RELEASE }} pydoc/current - + echo "............Copying generated typedoc into beam-site.........." mkdir -p typedoc cp -r ${BEAM_ROOT_DIR}/sdks/typescript/docs typedoc/${{ github.event.inputs.RELEASE }} # Update current symlink to point to the latest release unlink typedoc/current | true ln -s ${{ github.event.inputs.RELEASE }} typedoc/current + + echo "............Copying generated yamldoc into beam-site.........." + mkdir -p yamldoc + cp -r ${BEAM_ROOT_DIR}/sdks/python/build/yaml-ref.html yamldoc/${{ github.event.inputs.RELEASE }}/index.html + # Update current symlink to point to the latest release + unlink yamldoc/current | true + ln -s ${{ github.event.inputs.RELEASE }} yamldoc/current - name: Create commit on beam-site branch working-directory: beam run: | diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh index 745d726d7655..81425f46ed57 100755 --- a/release/src/main/scripts/build_release_candidate.sh +++ b/release/src/main/scripts/build_release_candidate.sh @@ -44,6 +44,7 @@ LOCAL_WEBSITE_UPDATE_DIR=website_update_dir LOCAL_PYTHON_DOC=python_doc LOCAL_TYPESCRIPT_DOC=typescript_doc LOCAL_JAVA_DOC=java_doc +LOCAL_YAML_DOC=yaml_doc LOCAL_WEBSITE_REPO=beam_website_repo USER_REMOTE_URL= @@ -333,6 +334,7 @@ if [[ $confirmation = "y" ]]; then mkdir -p ${LOCAL_PYTHON_DOC} mkdir -p ${LOCAL_TYPESCRIPT_DOC} mkdir -p ${LOCAL_JAVA_DOC} + mkdir -p ${LOCAL_YAML_DOC} mkdir -p ${LOCAL_WEBSITE_REPO} echo "------------------Building Python Doc------------------------" @@ -364,6 +366,13 @@ if [[ $confirmation = "y" ]]; then ./gradlew :sdks:java:javadoc:aggregateJavadoc -PisRelease --no-daemon --no-parallel GENERATE_JAVADOC=~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_JAVA_DOC}/${BEAM_ROOT_DIR}/sdks/java/javadoc/build/docs/javadoc/ + echo "----------------------Building YAML Doc----------------------" + cd ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_YAML_DOC} + git clone --branch "${RC_TAG}" --depth 1 ${GIT_REPO_URL} + cd ${BEAM_ROOT_DIR} + ./gradlew :sdks:python:generateYamlDocs -PisRelease + GENERATE_YAMLDOC=~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_YAML_DOC}/${BEAM_ROOT_DIR}/sdks/python/build/yaml-ref.html + echo "------------------Updating Release Docs---------------------" cd ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_WEBSITE_REPO} git clone ${GIT_BEAM_WEBSITE} @@ -390,6 +399,13 @@ if [[ $confirmation = "y" ]]; then unlink typedoc/current | true ln -s ${RELEASE} typedoc/current + echo "............Copying generated yaml docs into beam-site.........." + mkdir -p yamldoc + cp -r ${GENERATE_YAMLDOC} yamldoc/${RELEASE}/index.html + # Update current symlink to point to the latest release + unlink yamldoc/current | true + ln -s ${RELEASE} yamldoc/current + git add -A git commit -m "Update beam-site for release ${RELEASE}." -m "Content generated from commit ${RELEASE_COMMIT}." git push -f ${USER_REMOTE_URL} @@ -419,4 +435,5 @@ if [[ $confirmation = "y" ]]; then rm -rf ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_JAVA_DOC} rm -rf ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_PYTHON_DOC} rm -rf ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_TYPESCRIPT_DOC} + rm -rf ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_YAML_DOC} fi diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle index 85b26ca3a464..37b80519b7c4 100644 --- a/sdks/python/build.gradle +++ b/sdks/python/build.gradle @@ -80,11 +80,33 @@ tasks.register("generateExternalTransformsConfig") { exec { executable 'sh' args '-c', "pip install $PyYaml && " + - "python gen_xlang_wrappers.py --cleanup --generate-config-only" + "python -m apache_beam.yaml.generate_yaml_docs.py --cleanup --generate-config-only" } } } +tasks.register("generateYamlDocs") { + description "Generates the reference documentation for all YAML transforms." + + dependsOn buildPython + // Need to build all expansion services referenced in apache_beam/yaml/*.* + // grep -oh 'sdk.*Jar' sdks/python/apache_beam/yaml/*.yaml | sort | uniq + dependsOn ":sdks:java:extensions:schemaio-expansion-service:shadowJar" + dependsOn ":sdks:java:extensions:sql:expansion-service:shadowJar" + dependsOn ":sdks:java:io:expansion-service:build" + dependsOn ":sdks:java:io:google-cloud-platform:expansion-service:build" + def extraPackages = "pyyaml markdown docstring_parser pandas pygments" + + doLast { + exec { + executable 'sh' + args '-c', "${envdir}/bin/pip install $extraPackages && " + + "${envdir}/bin/python -m apache_beam.yaml.generate_yaml_docs --html_file=${buildDir}/yaml-ref.html" + } + } + outputs.file "${buildDir}/yaml-ref.html" +} + // Create Python wheels for given platform and Python version // build identifiers for cibuildwheel def platform_identifiers_map = [