Skip to content

Commit

Permalink
ZetaSQL test Java version fixes
Browse files Browse the repository at this point in the history
* Align SDK container version with pipeline submission env

* Disable ZetaSQL test on Java8
  • Loading branch information
Abacn committed Nov 25, 2024
1 parent d342dd3 commit 6bc6036
Show file tree
Hide file tree
Showing 5 changed files with 25 additions and 1 deletion.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{}
1 change: 1 addition & 0 deletions .github/trigger_files/beam_PostCommit_XVR_Samza.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{}
1 change: 1 addition & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@

## Breaking Changes

* Upgraded ZetaSQL to 2024.11.1 ([#32902](https://github.com/apache/beam/pull/32902)). Java11+ is now needed if Beam's ZetaSQL component is used.
* X behavior was changed ([#X](https://github.com/apache/beam/issues/X)).

## Deprecations
Expand Down
4 changes: 3 additions & 1 deletion runners/google-cloud-dataflow-java/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
* limitations under the License.
*/

import static org.apache.beam.gradle.BeamModulePlugin.getSupportedJavaVersion

import groovy.json.JsonOutput

plugins { id 'org.apache.beam.module' }
Expand Down Expand Up @@ -278,7 +280,7 @@ def createRunnerV2ValidatesRunnerTest = { Map args ->
// task directly ('dependsOn buildAndPushDockerJavaContainer'). This ensures the correct
// task ordering such that the registry doesn't get cleaned up prior to task completion.
def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaContainer") {
def javaVer = "java8"
def javaVer = getSupportedJavaVersion()
if(project.hasProperty('testJavaVersion')) {
javaVer = "java${project.getProperty('testJavaVersion')}"
}
Expand Down
19 changes: 19 additions & 0 deletions sdks/python/apache_beam/transforms/sql_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
# pytype: skip-file

import logging
import subprocess
import typing
import unittest

Expand Down Expand Up @@ -69,6 +70,21 @@ class SqlTransformTest(unittest.TestCase):
"""
_multiprocess_can_split_ = True

@staticmethod
def _disable_zetasql_test():
# disable if run on Java8 which is no longer supported by ZetaSQL
try:
result = subprocess.run(['java', '-version'],
capture_output=True,
text=True)
version_line = result.stderr.splitlines()[0]
version = version_line.split()[2].strip('\"')
if version.starts_with("1."):
return True
return False
except:
return False

def test_generate_data(self):
with TestPipeline() as p:
out = p | SqlTransform(
Expand Down Expand Up @@ -150,6 +166,9 @@ def test_row(self):
assert_that(out, equal_to([(1, 1), (4, 1), (100, 2)]))

def test_zetasql_generate_data(self):
if self._disable_zetasql_test():
raise unittest.SkipTest("ZetaSQL tests need Java11+")

with TestPipeline() as p:
out = p | SqlTransform(
"""SELECT
Expand Down

0 comments on commit 6bc6036

Please sign in to comment.