Skip to content

Commit

Permalink
Enable Java SDK Distroless container image variant
Browse files Browse the repository at this point in the history
  • Loading branch information
damondouglas committed Nov 20, 2024
1 parent fd6b133 commit 45aaec5
Show file tree
Hide file tree
Showing 4 changed files with 122 additions and 2 deletions.
5 changes: 4 additions & 1 deletion .github/trigger_files/beam_PostCommit_Java.json
Original file line number Diff line number Diff line change
@@ -1 +1,4 @@
{}
{
"comment": "Modify this file in a trivial way to cause this test suite to run",
"modification": 1
}
2 changes: 1 addition & 1 deletion .github/trigger_files/beam_PostCommit_Java_DataflowV2.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to run",
"modification": 1
"modification": 2
}
81 changes: 81 additions & 0 deletions runners/google-cloud-dataflow-java/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,87 @@ def createRunnerV2ValidatesRunnerTest = { Map args ->
}
}

tasks.register('googleCloudPlatformRunnerV2DistrolessIntegrationTest', Test.class) {
group = "verification"
dependsOn 'buildAndPushDistrolessContainerImage'
def javaVer = project.findProperty('testJavaVersion')
def repository = "us.gcr.io/apache-beam-testing/${System.getenv('USER')}"
def tag = project.findProperty('dockerTag')
def imageURL = "${repository}/beam_${javaVer}_sdk_distroless:${tag}"
def pipelineOptions = [
"--runner=TestDataflowRunner",
"--project=${gcpProject}",
"--region=${gcpRegion}",
"--tempRoot=${dataflowValidatesTempRoot}",
"--sdkContainerImage=${imageURL}",
"--experiments=use_unified_worker,use_runner_v2",
"--firestoreDb=${firestoreDb}",
]
systemProperty "beamTestPipelineOptions", JsonOutput.toJson(pipelineOptions)

include '**/*IT.class'
exclude '**/BigQueryIOStorageReadTableRowIT.class'
exclude '**/SpannerWriteIT.class'
exclude '**/*KmsKeyIT.class'
exclude '**/FhirIOReadIT.class'
exclude '**/DicomIOReadIT.class'
exclude '**/FhirIOWriteIT.class'
exclude '**/FhirIOLROIT.class'
exclude '**/FhirIOSearchIT.class'
exclude '**/FhirIOPatientEverythingIT.class'
// failing due to pane index not incrementing after Reshuffle:
// https://github.com/apache/beam/issues/28219
exclude '**/FileLoadsStreamingIT.class'

maxParallelForks 4
classpath = configurations.googleCloudPlatformIntegrationTest
testClassesDirs = files(project(":sdks:java:io:google-cloud-platform").sourceSets.test.output.classesDirs)
useJUnit {
filter {
// Only needs to run on direct runner
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigtable.BigtableWriteIT.testE2EBigtableWriteWithEmptyMutationFailures'
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigtable.BigtableWriteIT.testE2EBigtableWriteWithEmptyRowFailures'
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigtable.BigtableWriteIT.testE2EBigtableWriteWithInvalidTimestampFailures'
excludeTestsMatching 'org.apache.beam.sdk.io.gcp.bigtable.BigtableWriteIT.testE2EBigtableWriteWithOversizedQualifierFailures'
}
}
}

tasks.register('buildAndPushDistrolessContainerImage', Task.class) {
def allowed = ["java17", "java21"]
doLast {
def javaVer = project.findProperty('testJavaVersion')
if (!allowed.contains(javaVer)) {
throw new GradleException("testJavaVersion must be one of ${allowed}, got: ${javaVer}")
}
if (!project.hasProperty('dockerTag')) {
throw new GradleException("dockerTag is missing but required")
}
def repository = "us.gcr.io/apache-beam-testing/${System.getenv('USER')}"
def tag = project.findProperty('dockerTag')
def imageURL = "${repository}/beam_${javaVer}_sdk_distroless:${tag}"
exec {
executable 'docker'
workingDir rootDir
args = [
'buildx',
'build',
'-t',
imageURL,
'-f',
'sdks/java/container/Dockerfile-distroless',
"--build-arg=BEAM_BASE=gcr.io/apache-beam-testing/beam-sdk/beam_${javaVer}_sdk",
"--build-arg=DISTROLESS_BASE=gcr.io/distroless/${javaVer}-debian12",
'.'
]
}
exec {
executable 'docker'
args = ['push', imageURL]
}
}
}

// Push docker images to a container registry for use within tests.
// NB: Tasks which consume docker images from the registry should depend on this
// task directly ('dependsOn buildAndPushDockerJavaContainer'). This ensures the correct
Expand Down
36 changes: 36 additions & 0 deletions sdks/java/container/Dockerfile-distroless
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
###############################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################

# ARG BEAM_BASE is the Beam SDK container image built using sdks/python/container/Dockerfile.
ARG BEAM_BASE

# ARG DISTROLESS_BASE is the distroless container image URL. For available distroless Java images,
# see https://github.com/GoogleContainerTools/distroless/tree/main?tab=readme-ov-file#what-images-are-available.
# Only Java versions 17 and 21 are supported.
ARG DISTROLESS_BASE
FROM ${BEAM_BASE} AS base
ARG TARGETARCH
LABEL Author="Apache Beam <[email protected]>"

RUN if [ -z "${TARGETARCH}" ]; then echo "fatal: TARGETARCH not set; run as docker buildx build or use --build-arg=TARGETARCH=amd64|arm64" >&2; exit 1; fi

FROM ${DISTROLESS_BASE}:latest-${TARGETARCH} AS distroless

COPY --from=base /opt /opt

ENTRYPOINT ["/opt/apache/beam/boot"]

0 comments on commit 45aaec5

Please sign in to comment.