Merge branch 'main' into group_error_prone

googleapis · Dec 18, 2024 · af7a615 · af7a615
2 parents ebacb69 + ae493f0
commit af7a615
Show file tree

Hide file tree

Showing 222 changed files with 5,143 additions and 2,317 deletions.
diff --git a/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kms.yaml b/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kms.yaml
@@ -15,9 +15,10 @@
 timeout: 7200s # 2 hours
 substitutions:
   _SHARED_DEPENDENCIES_VERSION: '3.30.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current}
-  _JAVA_SHARED_CONFIG_VERSION: '1.11.3'
+  _JAVA_SHARED_CONFIG_VERSION: '1.12.2'
 options:
   machineType: 'E2_HIGHCPU_8'
+  logging: CLOUD_LOGGING_ONLY
 steps:
   - name: gcr.io/cloud-builders/docker
     args: [
@@ -44,4 +45,4 @@ steps:
     id: native-java-kms
     env:
       - 'MODULES_UNDER_TEST=java-kms'
-      - 'GOOGLE_CLOUD_PROJECT=cloud-java-ci-test'
+      - 'GOOGLE_CLOUD_PROJECT=java-graalvm-ci-prod'
diff --git a/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kmsinventory.yaml b/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kmsinventory.yaml
@@ -15,9 +15,10 @@
 timeout: 7200s # 2 hours
 substitutions:
   _SHARED_DEPENDENCIES_VERSION: '3.30.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current}
-  _JAVA_SHARED_CONFIG_VERSION: '1.11.3'
+  _JAVA_SHARED_CONFIG_VERSION: '1.12.2'
 options:
   machineType: 'E2_HIGHCPU_8'
+  logging: CLOUD_LOGGING_ONLY
 steps:
   - name: gcr.io/cloud-builders/docker
     args: [
@@ -44,4 +45,4 @@ steps:
     id: native-java-kmsinventory
     env:
       - 'MODULES_UNDER_TEST=java-kmsinventory'
-      - 'GOOGLE_CLOUD_PROJECT=cloud-java-ci-test'
+      - 'GOOGLE_CLOUD_PROJECT=java-graalvm-ci-prod'
diff --git a/.cloudbuild/graalvm/cloudbuild-test-a.yaml b/.cloudbuild/graalvm/cloudbuild-test-a.yaml
@@ -14,10 +14,11 @@
 
 timeout: 7200s # 2 hours
 substitutions:
-  _SHARED_DEPENDENCIES_VERSION: '3.39.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current}
-  _JAVA_SHARED_CONFIG_VERSION: '1.11.3'
+  _SHARED_DEPENDENCIES_VERSION: '3.41.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current}
+  _JAVA_SHARED_CONFIG_VERSION: '1.12.2'
 options:
   machineType: 'E2_HIGHCPU_8'
+  logging: CLOUD_LOGGING_ONLY
 steps:
   - name: gcr.io/cloud-builders/docker
     args: [
@@ -41,4 +42,5 @@ steps:
     entrypoint: bash
     args: [ './.kokoro/presubmit/showcase-native.sh' ]
     waitFor: [ "graalvm-a-build" ]
-    id: native-showcase
+    id: native-showcase
+
diff --git a/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kms.yaml b/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kms.yaml
@@ -15,9 +15,10 @@
 timeout: 7200s # 2 hours
 substitutions:
   _SHARED_DEPENDENCIES_VERSION: '3.30.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current}
-  _JAVA_SHARED_CONFIG_VERSION: '1.11.3'
+  _JAVA_SHARED_CONFIG_VERSION: '1.12.2'
 options:
   machineType: 'E2_HIGHCPU_8'
+  logging: CLOUD_LOGGING_ONLY
 steps:
   - name: gcr.io/cloud-builders/docker
     args: [
@@ -43,4 +44,4 @@ steps:
     waitFor: [ "graalvm-b-build" ]
     env:
       - 'MODULES_UNDER_TEST=java-kms'
-      - 'GOOGLE_CLOUD_PROJECT=cloud-java-ci-test'
+      - 'GOOGLE_CLOUD_PROJECT=java-graalvm-ci-prod'
diff --git a/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kmsinventory.yaml b/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kmsinventory.yaml
@@ -15,9 +15,10 @@
 timeout: 7200s # 2 hours
 substitutions:
   _SHARED_DEPENDENCIES_VERSION: '3.30.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current}
-  _JAVA_SHARED_CONFIG_VERSION: '1.11.3'
+  _JAVA_SHARED_CONFIG_VERSION: '1.12.2'
 options:
   machineType: 'E2_HIGHCPU_8'
+  logging: CLOUD_LOGGING_ONLY
 steps:
   - name: gcr.io/cloud-builders/docker
     args: [
@@ -44,4 +45,4 @@ steps:
     id: native-java-kmsinventory
     env:
       - 'MODULES_UNDER_TEST=java-kmsinventory'
-      - 'GOOGLE_CLOUD_PROJECT=cloud-java-ci-test'
+      - 'GOOGLE_CLOUD_PROJECT=java-graalvm-ci-prod'
diff --git a/.cloudbuild/graalvm/cloudbuild-test-b.yaml b/.cloudbuild/graalvm/cloudbuild-test-b.yaml
@@ -14,10 +14,11 @@
 
 timeout: 7200s # 2 hours
 substitutions:
-  _SHARED_DEPENDENCIES_VERSION: '3.39.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current}
-  _JAVA_SHARED_CONFIG_VERSION: '1.11.3'
+  _SHARED_DEPENDENCIES_VERSION: '3.41.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current}
+  _JAVA_SHARED_CONFIG_VERSION: '1.12.2'
 options:
   machineType: 'E2_HIGHCPU_8'
+  logging: CLOUD_LOGGING_ONLY
 steps:
   - name: gcr.io/cloud-builders/docker
     args: [

diff --git a/.cloudbuild/graalvm/cloudbuild.yaml b/.cloudbuild/graalvm/cloudbuild.yaml
@@ -14,8 +14,8 @@
 
 timeout: 7200s # 2 hours
 substitutions:
-  _SHARED_DEPENDENCIES_VERSION: '3.39.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current}
-  _JAVA_SHARED_CONFIG_VERSION: '1.11.3'
+  _SHARED_DEPENDENCIES_VERSION: '3.41.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current}
+  _JAVA_SHARED_CONFIG_VERSION: '1.12.2'
 steps:
   # GraalVM A build
   - name: gcr.io/cloud-builders/docker
@@ -45,6 +45,8 @@ steps:
     id: graalvm-b-build
     waitFor: [ "-" ]
 
+options:
+  logging: CLOUD_LOGGING_ONLY
 
 images:
   - gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:${_SHARED_DEPENDENCIES_VERSION}

diff --git a/.cloudbuild/graalvm/graalvm_a.Dockerfile b/.cloudbuild/graalvm/graalvm_a.Dockerfile
@@ -15,5 +15,5 @@
 
 ARG JAVA_SHARED_CONFIG_VERSION
 
-FROM gcr.io/cloud-devrel-public-resources/graalvm_a:$JAVA_SHARED_CONFIG_VERSION
+FROM us-docker.pkg.dev/java-graalvm-ci-prod/graalvm-integration-testing/graalvm_a:$JAVA_SHARED_CONFIG_VERSION
 
diff --git a/.cloudbuild/graalvm/graalvm_b.Dockerfile b/.cloudbuild/graalvm/graalvm_b.Dockerfile
@@ -15,4 +15,4 @@
 
 ARG JAVA_SHARED_CONFIG_VERSION
 
-FROM gcr.io/cloud-devrel-public-resources/graalvm_b:$JAVA_SHARED_CONFIG_VERSION
+FROM us-docker.pkg.dev/java-graalvm-ci-prod/graalvm-integration-testing/graalvm_b:$JAVA_SHARED_CONFIG_VERSION
diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-push-prod.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-push-prod.yaml
@@ -15,7 +15,7 @@
 timeout: 7200s # 2 hours
 substitutions:
   _IMAGE_NAME: "us-docker.pkg.dev/java-hermetic-build-prod/private-resources/java-library-generation"
-  _GAPIC_GENERATOR_JAVA_VERSION: '2.46.2-SNAPSHOT' # {x-version-update:gapic-generator-java:current}
+  _GAPIC_GENERATOR_JAVA_VERSION: '2.51.1-SNAPSHOT' # {x-version-update:gapic-generator-java:current}
   _SHA_IMAGE_ID: "${_IMAGE_NAME}:${COMMIT_SHA}"
   _LATEST_IMAGE_ID: "${_IMAGE_NAME}:latest"
   _VERSIONED_IMAGE_ID: "${_IMAGE_NAME}:${_GAPIC_GENERATOR_JAVA_VERSION}"
@@ -27,9 +27,13 @@ steps:
       "-t", "${_SHA_IMAGE_ID}",
       "-t", "${_LATEST_IMAGE_ID}",
       "-t", "${_VERSIONED_IMAGE_ID}",
-      "--file", ".cloudbuild/library_generation/library_generation.Dockerfile", "."]
+      "-f", ".cloudbuild/library_generation/library_generation_airlock.Dockerfile",
+      "."
+    ]
     id: library-generation-build
     waitFor: ["-"]
+    env: 
+      - 'DOCKER_BUILDKIT=1'
 
 options:
   logging: CLOUD_LOGGING_ONLY

diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-push.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-push.yaml
@@ -15,7 +15,7 @@
 timeout: 7200s # 2 hours
 substitutions:
   _IMAGE_NAME: "gcr.io/cloud-devrel-public-resources/java-library-generation"
-  _GAPIC_GENERATOR_JAVA_VERSION: '2.49.1-SNAPSHOT' # {x-version-update:gapic-generator-java:current}
+  _GAPIC_GENERATOR_JAVA_VERSION: '2.51.1-SNAPSHOT' # {x-version-update:gapic-generator-java:current}
   _SHA_IMAGE_ID: "${_IMAGE_NAME}:${COMMIT_SHA}"
   _LATEST_IMAGE_ID: "${_IMAGE_NAME}:latest"
   _VERSIONED_IMAGE_ID: "${_IMAGE_NAME}:${_GAPIC_GENERATOR_JAVA_VERSION}"
@@ -30,6 +30,8 @@ steps:
       "--file", ".cloudbuild/library_generation/library_generation.Dockerfile", "."]
     id: library-generation-build
     waitFor: ["-"]
+    env: 
+      - 'DOCKER_BUILDKIT=1'
 
 options:
   logging: CLOUD_LOGGING_ONLY

diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile
@@ -14,35 +14,70 @@
 
 # install gapic-generator-java in a separate layer so we don't overload the image
 # with the transferred source code and jars
-FROM gcr.io/cloud-devrel-public-resources/java21@sha256:2ceff5eeea72260258df56d42e44ed413e52ee421c1b77393c5f2c9c4d7c41da AS ggj-build
+
+FROM docker.io/library/maven:3.9.9-eclipse-temurin-11-alpine@sha256:3bab9f2cd4fa8704445bf01444e8e5e0f1ab819a41616069872a898b432a7309 AS ggj-build
 
 WORKDIR /sdk-platform-java
 COPY . .
 # {x-version-update-start:gapic-generator-java:current}
-ENV DOCKER_GAPIC_GENERATOR_VERSION="2.49.1-SNAPSHOT" 
+ENV DOCKER_GAPIC_GENERATOR_VERSION="2.51.1-SNAPSHOT"
 # {x-version-update-end}
 
 RUN mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip
 RUN cp "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" \
   "./gapic-generator-java.jar"
 
-# build from the root of this repo:
-FROM gcr.io/cloud-devrel-public-resources/python@sha256:9c5ea427632f195ad164054831968389d86fdde4a15abca651f3fcb2a71268cb
-
-SHELL [ "/bin/bash", "-c" ]
+FROM docker.io/library/alpine:3.21.0@sha256:21dc6063fd678b478f57c0e13f47560d0ea4eeba26dfc947b2a4f81f686b9f45 as glibc-compat
 
+RUN apk add git sudo
+# This SHA is the latest known-to-work version of this binary compatibility tool
+ARG GLIB_MUS_SHA=e94aca542e3ab08b42aa0b0d6e72478b935bb8e8
+WORKDIR /home
 
-ARG OWLBOT_CLI_COMMITTISH=38fe6f89a2339ee75c77739b31b371f601b01bb3
+# Install compatibility layer to run glibc-based programs (such as the
+# grpc plugin).
+# Alpine, by default, only supports musl-based binaries, and there is no public
+# downloadable distribution of the grpc plugin that is Alpine (musl) compatible.
+# This is one of the recommended approaches to ensure glibc-compatibility
+# as per https://wiki.alpinelinux.org/wiki/Running_glibc_programs
+RUN git clone https://gitlab.com/manoel-linux1/GlibMus-HQ.git
+WORKDIR /home/GlibMus-HQ
+# We lock the tool to the latest known-to-work version
+RUN git checkout "${GLIB_MUS_SHA}"
+RUN chmod a+x compile-x86_64-alpine-linux.sh
+RUN sh compile-x86_64-alpine-linux.sh
+
+FROM docker.io/library/python:3.13.1-alpine3.20@sha256:804ad02b9ba67ea1f8307eeb6407b121c6bd6bb19d3f182aae166821eb59d6a4 as final
+
+ARG OWLBOT_CLI_COMMITTISH=8b7d94b4a8ad0345aeefd6a7ec9c5afcbeb8e2d7
 ARG PROTOC_VERSION=25.5
-ARG GRPC_VERSION=1.67.1
+ARG GRPC_VERSION=1.69.0
 ARG JAVA_FORMAT_VERSION=1.7
 ENV HOME=/home
 ENV OS_ARCHITECTURE="linux-x86_64"
 
 # install OS tools
-RUN apt-get update && apt-get install -y \
-	unzip openjdk-17-jdk rsync maven jq \
-	&& apt-get clean
+RUN apk update && apk add unzip curl rsync openjdk11 jq bash nodejs npm git
+
+SHELL [ "/bin/bash", "-c" ]
+
+# Copy glibc shared objects to enable execution of the grpc plugin.
+# This list was obtained via `libtree -pvvv /grpc/*` in the final container as
+# well as inspecting the modifications done by compile-x86_64-alpine-linux.sh
+# in the glibc-compat stage using the `dive` command.
+COPY --from=glibc-compat /etc/libgcc* /etc/
+COPY --from=glibc-compat /lib64/ld-linux-x86-64.so.2 /lib64/
+COPY --from=glibc-compat /lib/GLIBCFAKE.so.0 /lib/
+COPY --from=glibc-compat /lib/ld-linux-x86-64.so.2 /lib/
+COPY --from=glibc-compat /lib/libpthread* /lib/
+COPY --from=glibc-compat /lib/libucontext* /lib/
+COPY --from=glibc-compat /lib/libc.* /lib/
+COPY --from=glibc-compat /usr/lib/libgcc* /usr/lib/
+COPY --from=glibc-compat /usr/lib/libstdc* /usr/lib/
+COPY --from=glibc-compat /usr/lib/libobstack* /usr/lib/
+COPY --from=glibc-compat /lib/libm.so.6 /usr/lib/
+COPY --from=glibc-compat /usr/lib/libucontext.so.1 /usr/lib/
+
 
 # copy source code
 COPY hermetic_build/common /src/common
@@ -53,29 +88,24 @@ WORKDIR /protoc
 RUN source /src/library_generation/utils/utilities.sh \
 	&& download_protoc "${PROTOC_VERSION}" "${OS_ARCHITECTURE}"
 # we indicate protoc is available in the container via env vars
-ENV DOCKER_PROTOC_LOCATION=/protoc
+ENV DOCKER_PROTOC_LOCATION=/protoc/bin
 ENV DOCKER_PROTOC_VERSION="${PROTOC_VERSION}"
 
 # install grpc
 WORKDIR /grpc
 RUN source /src/library_generation/utils/utilities.sh \
 	&& download_grpc_plugin "${GRPC_VERSION}" "${OS_ARCHITECTURE}"
 # similar to protoc, we indicate grpc is available in the container via env vars
-ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java-${GRPC_VERSION}-${OS_ARCHITECTURE}.exe"
-ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}"
-
+ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java.exe"
 
 # Here we transfer gapic-generator-java from the previous stage.
 # Note that the destination is a well-known location that will be assumed at runtime
 # We hard-code the location string to avoid making it configurable (via ARG) as
 # well as to avoid it making it overridable at runtime (via ENV).
 COPY --from=ggj-build "/sdk-platform-java/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar"
 RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar"
+ENV GAPIC_GENERATOR_LOCATION="${HOME}/.library_generation/gapic-generator-java.jar"
 
-#  use python 3.12 (the base image has several python versions; here we define the default one)
-RUN rm $(which python3)
-RUN ln -s $(which python3.12) /usr/local/bin/python
-RUN ln -s $(which python3.12) /usr/local/bin/python3
 RUN python -m pip install --upgrade pip
 
 # install main scripts as a python package
@@ -85,30 +115,21 @@ RUN python -m pip install src/common
 RUN python -m pip install --require-hashes -r src/library_generation/requirements.txt
 RUN python -m pip install src/library_generation
 
-# Install nvm with node and npm
-ENV NODE_VERSION 20.12.0
-WORKDIR /home
-RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash
-RUN chmod o+rx /home/.nvm
-ENV NODE_PATH=/home/.nvm/versions/node/v${NODE_VERSION}/bin
-ENV PATH=${PATH}:${NODE_PATH}
-RUN node --version
-RUN npm --version
-
 # install the owl-bot CLI
 WORKDIR /tools
 RUN git clone https://github.com/googleapis/repo-automation-bots
 WORKDIR /tools/repo-automation-bots/packages/owl-bot
 RUN git checkout "${OWLBOT_CLI_COMMITTISH}"
 RUN npm i && npm run compile && npm link
 RUN owl-bot copy-code --version
-RUN chmod -R o+rx ${NODE_PATH}
-RUN ln -sf ${NODE_PATH}/* /usr/local/bin
+RUN chmod o+rx $(which owl-bot)
+RUN apk del -r npm && apk cache clean
 
 # download the Java formatter
 ADD https://maven-central.storage-download.googleapis.com/maven2/com/google/googlejavaformat/google-java-format/${JAVA_FORMAT_VERSION}/google-java-format-${JAVA_FORMAT_VERSION}-all-deps.jar \
   "${HOME}"/.library_generation/google-java-format.jar
 RUN chmod 755 "${HOME}"/.library_generation/google-java-format.jar
+ENV JAVA_FORMATTER_LOCATION="${HOME}/.library_generation/google-java-format.jar"
 
 # allow users to access the script folders
 RUN chmod -R o+rx /src
@@ -120,7 +141,6 @@ RUN git config --system user.name "Cloud Java Bot"
 
 # allow read-write for /home and execution for binaries in /home/.nvm
 RUN chmod -R a+rw /home
-RUN chmod -R a+rx /home/.nvm
 
 WORKDIR /workspace
 ENTRYPOINT [ "python", "/src/library_generation/cli/entry_point.py", "generate" ]
Original file line number	Diff line number	Diff line change
Expand Up		@@ -15,5 +15,5 @@

		ARG JAVA_SHARED_CONFIG_VERSION

		FROM gcr.io/cloud-devrel-public-resources/graalvm_a:$JAVA_SHARED_CONFIG_VERSION
		FROM us-docker.pkg.dev/java-graalvm-ci-prod/graalvm-integration-testing/graalvm_a:$JAVA_SHARED_CONFIG_VERSION
Original file line number	Diff line number	Diff line change
Expand Up		@@ -15,4 +15,4 @@

		ARG JAVA_SHARED_CONFIG_VERSION

		FROM gcr.io/cloud-devrel-public-resources/graalvm_b:$JAVA_SHARED_CONFIG_VERSION
		FROM us-docker.pkg.dev/java-graalvm-ci-prod/graalvm-integration-testing/graalvm_b:$JAVA_SHARED_CONFIG_VERSION