Merge pull request apache#30741 Gradle target and script for adding Y…

…AML reference manual to site.
linkedin · Mar 28, 2024 · 067b8fb · 067b8fb
2 parents 069c045 + 29cadc3
commit 067b8fb
Show file tree

Hide file tree

Showing 3 changed files with 65 additions and 15 deletions.
diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml
@@ -89,8 +89,8 @@ jobs:
         run: git config credential.helper store
       - name: Stage Java Artifacts into Maven
         run: ./gradlew publish -Psigning.gnupg.keyName=${{steps.import_gpg.outputs.fingerprint}} -PisRelease --no-daemon --no-parallel
-        
-      
+
+
   stage_java_source:
     if: ${{github.event.inputs.STAGE_JAVA_SOURCE == 'yes'}}
     runs-on: ubuntu-latest
@@ -153,8 +153,8 @@ jobs:
 
             svn add --force .
             svn status
-            svn commit -m "Staging Java artifacts for Apache Beam ${{ github.event.inputs.RELEASE }} RC${{ github.event.inputs.RC }}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}" 
-        
+            svn commit -m "Staging Java artifacts for Apache Beam ${{ github.event.inputs.RELEASE }} RC${{ github.event.inputs.RC }}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}"
+
   stage_python_artifacts:
     if: ${{github.event.inputs.STAGE_PYTHON_ARTIFACTS == 'yes'}}
     runs-on: ubuntu-latest
@@ -204,39 +204,39 @@ jobs:
 
           git fetch --all --tags --prune
           RELEASE_COMMIT=$(git rev-list -n 1 "tags/${RC_TAG}")
-          
+
           python "${SCRIPT_DIR}/download_github_actions_artifacts.py" \
             --github-token-var GITHUB_TOKEN \
             --repo-url "apache/beam" \
             --rc-tag "${RC_TAG}" \
             --release-commit "${RELEASE_COMMIT}" \
             --artifacts_dir "${RELEASE_DIR}/python" \
             --yes True
-        
+
           cd "${RELEASE_DIR}"/python
           ls
 
           echo "------Checking Hash Value for apache-beam-${RELEASE}.tar.gz-----"
           sha512sum -c "apache-beam-${RELEASE}.tar.gz.sha512"
-        
+
           echo "------Signing Source Release apache-beam-${RELEASE}.tar.gz------"
           gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --detach-sig "apache-beam-${RELEASE}.tar.gz"
-        
+
           for artifact in *.whl; do
             echo "----------Checking Hash Value for ${artifact} wheel-----------"
             sha512sum -c "${artifact}.sha512"
           done
-        
+
           for artifact in *.whl; do
             echo "------------------Signing ${artifact} wheel-------------------"
             gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --batch --yes --detach-sig "${artifact}"
           done
-        
+
           cd ..
           svn add --force python
           svn status
           svn commit -m "Staging Python artifacts for Apache Beam ${RELEASE} RC${RC_NUM}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}"
-        
+
 
   stage_docker:
     if: ${{github.event.inputs.STAGE_DOCKER_ARTIFACTS == 'yes'}}
@@ -343,6 +343,10 @@ jobs:
         working-directory: beam
         run: |
             ./gradlew :sdks:java:javadoc:aggregateJavadoc -PisRelease --no-daemon --no-parallel
+      - name: Build YAML Docs
+        working-directory: beam
+        run: |
+            ./gradlew :sdks:python:generateYamlDocs -PisRelease
       - name: Add canonical link into javadocs
         uses: cicirello/javadoc-cleanup@v1
         with:
@@ -352,25 +356,32 @@ jobs:
         working-directory: beam-site
         run: |
             git checkout -b $BRANCH_NAME release-docs
-             
+
             echo "..........Copying generated javadoc into beam-site.........."
             cp -r ${BEAM_ROOT_DIR}/sdks/java/javadoc/build/docs/javadoc/ javadoc/${{ github.event.inputs.RELEASE }}
             # Update current symlink to point to the latest release
             unlink javadoc/current
             ln -s ${{ github.event.inputs.RELEASE }} javadoc/current
-             
+
             echo "............Copying generated pydoc into beam-site.........."
             cp -r ${BEAM_ROOT_DIR}/sdks/python/target/docs/_build pydoc/${{  github.event.inputs.RELEASE }}
             # Update current symlink to point to the latest release
             unlink pydoc/current
             ln -s ${{ github.event.inputs.RELEASE }} pydoc/current
-             
+
             echo "............Copying generated typedoc into beam-site.........."
             mkdir -p typedoc
             cp -r ${BEAM_ROOT_DIR}/sdks/typescript/docs typedoc/${{ github.event.inputs.RELEASE }}
             # Update current symlink to point to the latest release
             unlink typedoc/current | true
             ln -s ${{ github.event.inputs.RELEASE }} typedoc/current
+
+            echo "............Copying generated yamldoc into beam-site.........."
+            mkdir -p yamldoc
+            cp -r ${BEAM_ROOT_DIR}/sdks/python/build/yaml-ref.html yamldoc/${{ github.event.inputs.RELEASE }}/index.html
+            # Update current symlink to point to the latest release
+            unlink yamldoc/current | true
+            ln -s ${{ github.event.inputs.RELEASE }} yamldoc/current
       - name: Create commit on beam-site branch
         working-directory: beam
         run: |

diff --git a/release/src/main/scripts/build_release_candidate.sh b/release/src/main/scripts/build_release_candidate.sh
@@ -44,6 +44,7 @@ LOCAL_WEBSITE_UPDATE_DIR=website_update_dir
 LOCAL_PYTHON_DOC=python_doc
 LOCAL_TYPESCRIPT_DOC=typescript_doc
 LOCAL_JAVA_DOC=java_doc
+LOCAL_YAML_DOC=yaml_doc
 LOCAL_WEBSITE_REPO=beam_website_repo
 
 USER_REMOTE_URL=
@@ -333,6 +334,7 @@ if [[ $confirmation = "y" ]]; then
   mkdir -p ${LOCAL_PYTHON_DOC}
   mkdir -p ${LOCAL_TYPESCRIPT_DOC}
   mkdir -p ${LOCAL_JAVA_DOC}
+  mkdir -p ${LOCAL_YAML_DOC}
   mkdir -p ${LOCAL_WEBSITE_REPO}
 
   echo "------------------Building Python Doc------------------------"
@@ -364,6 +366,13 @@ if [[ $confirmation = "y" ]]; then
   ./gradlew :sdks:java:javadoc:aggregateJavadoc -PisRelease --no-daemon --no-parallel
   GENERATE_JAVADOC=~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_JAVA_DOC}/${BEAM_ROOT_DIR}/sdks/java/javadoc/build/docs/javadoc/
 
+  echo "----------------------Building YAML Doc----------------------"
+  cd ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_YAML_DOC}
+  git clone --branch "${RC_TAG}" --depth 1 ${GIT_REPO_URL}
+  cd ${BEAM_ROOT_DIR}
+  ./gradlew :sdks:python:generateYamlDocs -PisRelease
+  GENERATE_YAMLDOC=~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_YAML_DOC}/${BEAM_ROOT_DIR}/sdks/python/build/yaml-ref.html
+
   echo "------------------Updating Release Docs---------------------"
   cd ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_WEBSITE_REPO}
   git clone ${GIT_BEAM_WEBSITE}
@@ -390,6 +399,13 @@ if [[ $confirmation = "y" ]]; then
   unlink typedoc/current | true
   ln -s ${RELEASE} typedoc/current
 
+  echo "............Copying generated yaml docs into beam-site.........."
+  mkdir -p yamldoc
+  cp -r ${GENERATE_YAMLDOC} yamldoc/${RELEASE}/index.html
+  # Update current symlink to point to the latest release
+  unlink yamldoc/current | true
+  ln -s ${RELEASE} yamldoc/current
+
   git add -A
   git commit -m "Update beam-site for release ${RELEASE}." -m "Content generated from commit ${RELEASE_COMMIT}."
   git push -f ${USER_REMOTE_URL}
@@ -419,4 +435,5 @@ if [[ $confirmation = "y" ]]; then
   rm -rf ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_JAVA_DOC}
   rm -rf ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_PYTHON_DOC}
   rm -rf ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_TYPESCRIPT_DOC}
+  rm -rf ~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_YAML_DOC}
 fi
diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
@@ -80,11 +80,33 @@ tasks.register("generateExternalTransformsConfig") {
     exec {
       executable 'sh'
       args '-c', "pip install $PyYaml && " +
-              "python gen_xlang_wrappers.py --cleanup --generate-config-only"
+              "python -m apache_beam.yaml.generate_yaml_docs.py --cleanup --generate-config-only"
     }
   }
 }
 
+tasks.register("generateYamlDocs") {
+  description "Generates the reference documentation for all YAML transforms."
+
+  dependsOn buildPython
+  // Need to build all expansion services referenced in apache_beam/yaml/*.*
+  // grep -oh 'sdk.*Jar' sdks/python/apache_beam/yaml/*.yaml | sort | uniq
+  dependsOn ":sdks:java:extensions:schemaio-expansion-service:shadowJar"
+  dependsOn ":sdks:java:extensions:sql:expansion-service:shadowJar"
+  dependsOn ":sdks:java:io:expansion-service:build"
+  dependsOn ":sdks:java:io:google-cloud-platform:expansion-service:build"
+  def extraPackages = "pyyaml markdown docstring_parser pandas pygments"
+
+  doLast {
+    exec {
+      executable 'sh'
+      args '-c', "${envdir}/bin/pip install $extraPackages && " +
+              "${envdir}/bin/python -m apache_beam.yaml.generate_yaml_docs --html_file=${buildDir}/yaml-ref.html"
+    }
+  }
+  outputs.file "${buildDir}/yaml-ref.html"
+}
+
 // Create Python wheels for given platform and Python version
 // build identifiers for cibuildwheel
 def platform_identifiers_map = [