diff --git a/.github/trigger_files/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.json b/.github/trigger_files/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.json new file mode 100644 index 000000000000..9e26dfeeb6e6 --- /dev/null +++ b/.github/trigger_files/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/.github/trigger_files/beam_PostCommit_XVR_Samza.json b/.github/trigger_files/beam_PostCommit_XVR_Samza.json new file mode 100644 index 000000000000..9e26dfeeb6e6 --- /dev/null +++ b/.github/trigger_files/beam_PostCommit_XVR_Samza.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/CHANGES.md b/CHANGES.md index 979cbbd67329..9a80678821d9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -70,6 +70,7 @@ ## Breaking Changes +* Upgraded ZetaSQL to 2024.11.1 ([#32902](https://github.com/apache/beam/pull/32902)). Java11+ is now needed if Beam's ZetaSQL component is used. * X behavior was changed ([#X](https://github.com/apache/beam/issues/X)). ## Deprecations diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 4906d9cf9cb8..2dcfecb3bea2 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -16,6 +16,8 @@ * limitations under the License. */ +import static org.apache.beam.gradle.BeamModulePlugin.getSupportedJavaVersion + import groovy.json.JsonOutput plugins { id 'org.apache.beam.module' } @@ -278,7 +280,7 @@ def createRunnerV2ValidatesRunnerTest = { Map args -> // task directly ('dependsOn buildAndPushDockerJavaContainer'). This ensures the correct // task ordering such that the registry doesn't get cleaned up prior to task completion. def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaContainer") { - def javaVer = "java8" + def javaVer = getSupportedJavaVersion() if(project.hasProperty('testJavaVersion')) { javaVer = "java${project.getProperty('testJavaVersion')}" } diff --git a/sdks/python/apache_beam/transforms/sql_test.py b/sdks/python/apache_beam/transforms/sql_test.py index 854aec078ce5..a7da253c4617 100644 --- a/sdks/python/apache_beam/transforms/sql_test.py +++ b/sdks/python/apache_beam/transforms/sql_test.py @@ -20,6 +20,7 @@ # pytype: skip-file import logging +import subprocess import typing import unittest @@ -69,6 +70,22 @@ class SqlTransformTest(unittest.TestCase): """ _multiprocess_can_split_ = True + @staticmethod + def _disable_zetasql_test(): + # disable if run on Java8 which is no longer supported by ZetaSQL + try: + result = subprocess.run(['java', '-version'], + check=True, + capture_output=True, + text=True) + version_line = result.stderr.splitlines()[0] + version = version_line.split()[2].strip('\"') + if version.startswith("1."): + return True + return False + except: # pylint: disable=bare-except + return False + def test_generate_data(self): with TestPipeline() as p: out = p | SqlTransform( @@ -150,6 +167,9 @@ def test_row(self): assert_that(out, equal_to([(1, 1), (4, 1), (100, 2)])) def test_zetasql_generate_data(self): + if self._disable_zetasql_test(): + raise unittest.SkipTest("ZetaSQL tests need Java11+") + with TestPipeline() as p: out = p | SqlTransform( """SELECT