diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtils.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtils.java index 61a0339a64c3..c169a0571b79 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtils.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtils.java @@ -532,11 +532,11 @@ private static Field convertField( (Object) null /* Cast to avoid deprecated JsonNode constructor. */); } - static TableSchema toTableSchema(Schema schema) { - return toTableSchema(schema, true); + static TableSchema fromGenericAvroSchema(Schema schema) { + return fromGenericAvroSchema(schema, true); } - static TableSchema toTableSchema(Schema schema, Boolean useAvroLogicalTypes) { + static TableSchema fromGenericAvroSchema(Schema schema, Boolean useAvroLogicalTypes) { verify( schema.getType() == Type.RECORD, "Expected Avro schema type RECORD, not %s", @@ -544,12 +544,12 @@ static TableSchema toTableSchema(Schema schema, Boolean useAvroLogicalTypes) { List fields = schema.getFields().stream() - .map(f -> toTableFieldSchema(f, useAvroLogicalTypes)) + .map(f -> fromAvroFieldSchema(f, useAvroLogicalTypes)) .collect(Collectors.toList()); return new TableSchema().setFields(fields); } - private static TableFieldSchema toTableFieldSchema( + private static TableFieldSchema fromAvroFieldSchema( Schema.Field avrofield, Boolean useAvroLogicalTypes) { Schema fieldSchema = avrofield.schema(); TableFieldSchema field; @@ -660,7 +660,7 @@ private static TableFieldSchema typedTableFieldSchema(Schema type, Boolean useAv case RECORD: List recordFields = type.getFields().stream() - .map(f -> toTableFieldSchema(f, useAvroLogicalTypes)) + .map(f -> fromAvroFieldSchema(f, useAvroLogicalTypes)) .collect(Collectors.toList()); return new TableFieldSchema().setType("RECORD").setFields(recordFields); default: diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java index a7accfef48e0..2ff4a223cb30 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtils.java @@ -461,14 +461,14 @@ public static org.apache.avro.Schema toGenericAvroSchema(TableSchema tableSchema } /** Convert an Avro {@link org.apache.avro.Schema} to a BigQuery {@link TableSchema}. */ - public static TableSchema toTableSchema(org.apache.avro.Schema schema) { - return toTableSchema(schema, false); + public static TableSchema fromGenericAvroSchema(org.apache.avro.Schema schema) { + return fromGenericAvroSchema(schema, false); } /** Convert an Avro {@link org.apache.avro.Schema} to a BigQuery {@link TableSchema}. */ - public static TableSchema toTableSchema( + public static TableSchema fromGenericAvroSchema( org.apache.avro.Schema schema, Boolean useAvroLogicalTypes) { - return BigQueryAvroUtils.toTableSchema(schema, useAvroLogicalTypes); + return BigQueryAvroUtils.fromGenericAvroSchema(schema, useAvroLogicalTypes); } /** Convert a BigQuery {@link TableSchema} to Avro {@link org.apache.avro.Schema}. */ diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java index d00c1ba95027..9b752055d011 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java @@ -698,7 +698,7 @@ public void testConvertAvroSchemaToBigQuerySchema() { Schema avroSchema = avroSchema(f -> f.type().booleanType().noDefault()); TableSchema expected = tableSchema(f -> f.setType("BOOLEAN").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); } { @@ -706,7 +706,7 @@ public void testConvertAvroSchemaToBigQuerySchema() { Schema avroSchema = avroSchema(f -> f.type().optional().booleanType()); TableSchema expected = tableSchema(f -> f.setType("BOOLEAN").setMode("NULLABLE")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); } { @@ -714,7 +714,7 @@ public void testConvertAvroSchemaToBigQuerySchema() { Schema avroSchema = avroSchema(f -> f.type().array().items().booleanType().noDefault()); TableSchema expected = tableSchema(f -> f.setType("BOOLEAN").setMode("REPEATED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); } { @@ -722,7 +722,7 @@ public void testConvertAvroSchemaToBigQuerySchema() { Schema avroSchema = avroSchema(f -> f.type().intType().noDefault()); TableSchema expected = tableSchema(f -> f.setType("INTEGER").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); } { @@ -730,7 +730,7 @@ public void testConvertAvroSchemaToBigQuerySchema() { Schema avroSchema = avroSchema(f -> f.type().longType().noDefault()); TableSchema expected = tableSchema(f -> f.setType("INTEGER").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); } { @@ -738,7 +738,7 @@ public void testConvertAvroSchemaToBigQuerySchema() { Schema avroSchema = avroSchema(f -> f.type().floatType().noDefault()); TableSchema expected = tableSchema(f -> f.setType("FLOAT").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); } { @@ -746,7 +746,7 @@ public void testConvertAvroSchemaToBigQuerySchema() { Schema avroSchema = avroSchema(f -> f.type().doubleType().noDefault()); TableSchema expected = tableSchema(f -> f.setType("FLOAT").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); } { @@ -754,7 +754,7 @@ public void testConvertAvroSchemaToBigQuerySchema() { Schema avroSchema = avroSchema(f -> f.type().bytesType().noDefault()); TableSchema expected = tableSchema(f -> f.setType("BYTES").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); } { @@ -762,7 +762,7 @@ public void testConvertAvroSchemaToBigQuerySchema() { Schema avroSchema = avroSchema(f -> f.type().stringType().noDefault()); TableSchema expected = tableSchema(f -> f.setType("STRING").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); } { @@ -773,8 +773,8 @@ public void testConvertAvroSchemaToBigQuerySchema() { TableSchema expected = tableSchema(f -> f.setType("NUMERIC").setMode("REQUIRED")); TableSchema expectedRaw = tableSchema(f -> f.setType("BYTES").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); - assertEquals(expectedRaw, BigQueryAvroUtils.toTableSchema(avroSchema, false)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); + assertEquals(expectedRaw, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema, false)); } { @@ -786,8 +786,8 @@ public void testConvertAvroSchemaToBigQuerySchema() { tableSchema(f -> f.setType("NUMERIC").setPrecision(29L).setMode("REQUIRED")); TableSchema expectedRaw = tableSchema(f -> f.setType("BYTES").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); - assertEquals(expectedRaw, BigQueryAvroUtils.toTableSchema(avroSchema, false)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); + assertEquals(expectedRaw, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema, false)); } { @@ -799,8 +799,8 @@ public void testConvertAvroSchemaToBigQuerySchema() { tableSchema(f -> f.setType("NUMERIC").setPrecision(10L).setScale(9L).setMode("REQUIRED")); TableSchema expectedRaw = tableSchema(f -> f.setType("BYTES").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); - assertEquals(expectedRaw, BigQueryAvroUtils.toTableSchema(avroSchema, false)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); + assertEquals(expectedRaw, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema, false)); } { @@ -811,8 +811,8 @@ public void testConvertAvroSchemaToBigQuerySchema() { TableSchema expected = tableSchema(f -> f.setType("BIGNUMERIC").setMode("REQUIRED")); TableSchema expectedRaw = tableSchema(f -> f.setType("BYTES").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); - assertEquals(expectedRaw, BigQueryAvroUtils.toTableSchema(avroSchema, false)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); + assertEquals(expectedRaw, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema, false)); } { @@ -824,8 +824,8 @@ public void testConvertAvroSchemaToBigQuerySchema() { tableSchema(f -> f.setType("BIGNUMERIC").setPrecision(38L).setMode("REQUIRED")); TableSchema expectedRaw = tableSchema(f -> f.setType("BYTES").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); - assertEquals(expectedRaw, BigQueryAvroUtils.toTableSchema(avroSchema, false)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); + assertEquals(expectedRaw, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema, false)); } { @@ -838,8 +838,8 @@ public void testConvertAvroSchemaToBigQuerySchema() { f -> f.setType("BIGNUMERIC").setPrecision(39L).setScale(38L).setMode("REQUIRED")); TableSchema expectedRaw = tableSchema(f -> f.setType("BYTES").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); - assertEquals(expectedRaw, BigQueryAvroUtils.toTableSchema(avroSchema, false)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); + assertEquals(expectedRaw, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema, false)); } { @@ -849,8 +849,8 @@ public void testConvertAvroSchemaToBigQuerySchema() { TableSchema expected = tableSchema(f -> f.setType("DATE").setMode("REQUIRED")); TableSchema expectedRaw = tableSchema(f -> f.setType("INTEGER").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); - assertEquals(expectedRaw, BigQueryAvroUtils.toTableSchema(avroSchema, false)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); + assertEquals(expectedRaw, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema, false)); } { @@ -860,8 +860,8 @@ public void testConvertAvroSchemaToBigQuerySchema() { TableSchema expected = tableSchema(f -> f.setType("TIME").setMode("REQUIRED")); TableSchema expectedRaw = tableSchema(f -> f.setType("INTEGER").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); - assertEquals(expectedRaw, BigQueryAvroUtils.toTableSchema(avroSchema, false)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); + assertEquals(expectedRaw, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema, false)); } { @@ -871,8 +871,8 @@ public void testConvertAvroSchemaToBigQuerySchema() { TableSchema expected = tableSchema(f -> f.setType("TIME").setMode("REQUIRED")); TableSchema expectedRaw = tableSchema(f -> f.setType("INTEGER").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); - assertEquals(expectedRaw, BigQueryAvroUtils.toTableSchema(avroSchema, false)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); + assertEquals(expectedRaw, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema, false)); } { @@ -883,8 +883,8 @@ public void testConvertAvroSchemaToBigQuerySchema() { TableSchema expected = tableSchema(f -> f.setType("DATETIME").setMode("REQUIRED")); TableSchema expectedRaw = tableSchema(f -> f.setType("INTEGER").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); - assertEquals(expectedRaw, BigQueryAvroUtils.toTableSchema(avroSchema, false)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); + assertEquals(expectedRaw, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema, false)); } { @@ -895,8 +895,8 @@ public void testConvertAvroSchemaToBigQuerySchema() { TableSchema expected = tableSchema(f -> f.setType("DATETIME").setMode("REQUIRED")); TableSchema expectedRaw = tableSchema(f -> f.setType("INTEGER").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); - assertEquals(expectedRaw, BigQueryAvroUtils.toTableSchema(avroSchema, false)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); + assertEquals(expectedRaw, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema, false)); } { @@ -907,8 +907,8 @@ public void testConvertAvroSchemaToBigQuerySchema() { TableSchema expected = tableSchema(f -> f.setType("TIMESTAMP").setMode("REQUIRED")); TableSchema expectedRaw = tableSchema(f -> f.setType("INTEGER").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); - assertEquals(expectedRaw, BigQueryAvroUtils.toTableSchema(avroSchema, false)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); + assertEquals(expectedRaw, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema, false)); } { @@ -919,8 +919,8 @@ public void testConvertAvroSchemaToBigQuerySchema() { TableSchema expected = tableSchema(f -> f.setType("TIMESTAMP").setMode("REQUIRED")); TableSchema expectedRaw = tableSchema(f -> f.setType("INTEGER").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); - assertEquals(expectedRaw, BigQueryAvroUtils.toTableSchema(avroSchema, false)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); + assertEquals(expectedRaw, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema, false)); } { @@ -931,8 +931,8 @@ public void testConvertAvroSchemaToBigQuerySchema() { TableSchema expected = tableSchema(f -> f.setType("GEOGRAPHY").setMode("REQUIRED")); TableSchema expectedRaw = tableSchema(f -> f.setType("STRING").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); - assertEquals(expectedRaw, BigQueryAvroUtils.toTableSchema(avroSchema, false)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); + assertEquals(expectedRaw, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema, false)); } { @@ -942,8 +942,8 @@ public void testConvertAvroSchemaToBigQuerySchema() { TableSchema expected = tableSchema(f -> f.setType("JSON").setMode("REQUIRED")); TableSchema expectedRaw = tableSchema(f -> f.setType("STRING").setMode("REQUIRED")); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); - assertEquals(expectedRaw, BigQueryAvroUtils.toTableSchema(avroSchema, false)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); + assertEquals(expectedRaw, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema, false)); } { @@ -973,7 +973,7 @@ public void testConvertAvroSchemaToBigQuerySchema() { .setMode("REQUIRED") .setFields(Lists.newArrayList(subInteger, subFloat))); - assertEquals(expected, BigQueryAvroUtils.toTableSchema(avroSchema)); + assertEquals(expected, BigQueryAvroUtils.fromGenericAvroSchema(avroSchema)); } } diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java index e7d7ed23e6a1..f66a95c12965 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java @@ -4111,7 +4111,7 @@ public void testUpsertAndDeleteGenericRecords() throws Exception { .requiredString("sqn") .endRecord(); - TableSchema tableSchema = BigQueryAvroUtils.toTableSchema(avroSchema); + TableSchema tableSchema = BigQueryAvroUtils.fromGenericAvroSchema(avroSchema); Table fakeTable = new Table(); TableReference ref =