Skip to content

Commit

Permalink
[apache#24292] Create Avro extension for Java SDK
Browse files Browse the repository at this point in the history
  • Loading branch information
aromanenko-dev committed Dec 26, 2022
1 parent 820839f commit 873e0d0
Show file tree
Hide file tree
Showing 35 changed files with 11,647 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ protected abstract static class Block<T> {
* byte of the block is within the range {@code [start, end)}.
*/
@Experimental(Kind.SOURCE_SINK)
protected abstract static class BlockBasedReader<T> extends FileBasedReader<T> {
public abstract static class BlockBasedReader<T> extends FileBasedReader<T> {
private boolean atSplitPoint;

protected BlockBasedReader(BlockBasedSource<T> source) {
Expand Down Expand Up @@ -195,7 +195,7 @@ public final T getCurrent() throws NoSuchElementException {
* block boundaries.
*/
@Override
protected boolean isAtSplitPoint() {
public boolean isAtSplitPoint() {
return atSplitPoint;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ public static DefaultFilenamePolicy fromParams(Params params) {
* ".txt", with shardNum = 1 and numShards = 100, the following is produced:
* "path/to/output-001-of-100.txt".
*/
static ResourceId constructName(
public static ResourceId constructName(
ResourceId baseFilename,
String shardTemplate,
String suffix,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
public class ReadAllViaFileBasedSource<T>
extends PTransform<PCollection<ReadableFile>, PCollection<T>> {

protected static final boolean DEFAULT_USES_RESHUFFLE = true;
public static final boolean DEFAULT_USES_RESHUFFLE = true;
private final long desiredBundleSizeBytes;
private final SerializableFunction<String, ? extends FileBasedSource<T>> createSource;
private final Coder<T> coder;
Expand Down
56 changes: 56 additions & 0 deletions sdks/java/extensions/avro/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* License); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an AS IS BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

plugins { id 'org.apache.beam.module' }
applyJavaNature(
automaticModuleName: 'org.apache.beam.sdk.extensions.avro',
disableLintWarnings: ['rawtypes'], // Avro-generated test code has raw-type errors
publish: false,
exportJavadoc: false,
)
applyAvroNature()

description = "Apache Beam :: SDKs :: Java :: Extensions :: Avro"

// Exclude tests that need a runner
test {
systemProperty "beamUseDummyRunner", "true"
useJUnit {
excludeCategories "org.apache.beam.sdk.testing.NeedsRunner"
}
}

dependencies {
implementation library.java.byte_buddy
implementation library.java.vendored_guava_26_0_jre
implementation (project(path: ":sdks:java:core", configuration: "shadow")) {
// Exclude Avro dependencies from "core" since Avro support moved to this extension
exclude group: "org.apache.avro", module: "avro"
}
implementation library.java.error_prone_annotations
implementation library.java.avro
implementation library.java.joda_time
testImplementation (project(path: ":sdks:java:core", configuration: "shadowTest")) {
// Exclude Avro dependencies from "core" since Avro support moved to this extension
exclude group: "org.apache.avro", module: "avro"
}
testImplementation library.java.avro_tests
testImplementation library.java.junit
testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow")
testRuntimeOnly library.java.slf4j_jdk14
}
Loading

0 comments on commit 873e0d0

Please sign in to comment.