Skip to content

Commit

Permalink
Log offending union values (#15)
Browse files Browse the repository at this point in the history
* Add caching in ci

* Log offending values

* Update readme
  • Loading branch information
tuliren authored May 3, 2022
1 parent 51079af commit e007826
Show file tree
Hide file tree
Showing 4 changed files with 54 additions and 13 deletions.
13 changes: 12 additions & 1 deletion .github/workflows/java_ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ on: [push]
jobs:
build:
runs-on: ubuntu-latest
env:
GRADLE_OPTS: '-Dfile.encoding=utf-8 -Dorg.gradle.daemon=false'

steps:
- uses: actions/checkout@v2
Expand All @@ -16,8 +18,17 @@ jobs:
java-version: '14'
- name: Validate Gradle wrapper
uses: gradle/wrapper-validation-action@v1
- name: Set up cache
uses: actions/cache@v2
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
restore-keys: |
${{ runner.os }}-gradle-
- name: Build with Gradle
run: ./gradlew build
run: ./gradlew clean build
- name: Cleanup Gradle Cache
# Remove some files from the Gradle cache, so they aren't cached by GitHub Actions.
# Restoring these files from a GitHub Actions cache might cause problems for future builds.
Expand Down
29 changes: 28 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,9 +1,36 @@
# json-avro-converter

[![Java CI](https://github.com/airbytehq/json-avro-converter/actions/workflows/java_ci.yaml/badge.svg)](https://github.com/airbytehq/json-avro-converter/actions/workflows/java_ci.yaml)
[![Java CI](https://github.com/airbytehq/json-avro-converter/actions/workflows/java_ci.yaml/badge.svg)](https://github.com/airbytehq/json-avro-converter/actions/workflows/java_ci.yaml) [![JitPack](https://jitpack.io/v/airbytehq/json-avro-converter.svg)](https://jitpack.io/#airbytehq/json-avro-converter)

This is a Json to Avro object converter used by Airbyte. It is based on [json-avro-converter](https://github.com/allegro/json-avro-converter) from [Allegro Tech](https://github.com/allegro). For basic usage, see the README in the original repo.

## Install
This library is available on JitPack. Check [here](https://jitpack.io/#airbytehq/json-avro-converter) for details.

1. Add the JitPack repository to the root `build.gradle`:
```groovy
allprojects {
repositories {
maven { url 'https://jitpack.io' }
}
}
```
2. Add the dependency
```groovy
dependencies {
implementation 'com.github.airbytehq:json-avro-converter:<version>'
}
```

## Publish
To publish a new version, tag the desired commit with a new version label.

```bash
VERSION=1.0.0
git tag -a "${VERSION}" -m "Version ${VERSION}"
git push origin "${VERSION}"
```

## Airbyte Specific Features

### Fluent Constructor
Expand Down
Original file line number Diff line number Diff line change
@@ -1,39 +1,41 @@
package tech.allegro.schema.json2avro.converter;

import static java.util.stream.Collectors.*;

import java.util.Deque;

import org.apache.avro.AvroTypeException;

class AvroTypeExceptions {
static AvroTypeException enumException(Deque<String> fieldPath, String expectedSymbols) {
static AvroTypeException enumException(Deque<String> fieldPath, String expectedSymbols, Object offendingValue) {
return new AvroTypeException(new StringBuilder()
.append("Field ")
.append(PathsPrinter.print(fieldPath))
.append(" is expected to be of enum type and be one of ")
.append(expectedSymbols)
.append(", but it is: ")
.append(offendingValue)
.toString());
}

static AvroTypeException unionException(String fieldName, String expectedTypes, Deque<String> offendingPath) {
static AvroTypeException unionException(String fieldName, String expectedTypes, Deque<String> offendingPath, Object offendingValue) {
return new AvroTypeException(new StringBuilder()
.append("Could not evaluate union, field ")
.append(fieldName)
.append(" is expected to be one of these: ")
.append(expectedTypes)
.append(". If this is a complex type, check if offending field: ")
.append(". If this is a complex type, check if offending field (path: ")
.append(PathsPrinter.print(offendingPath))
.append(" adheres to schema.")
.append(") adheres to schema: ")
.append(offendingValue)
.toString());
}

static AvroTypeException typeException(Deque<String> fieldPath, String expectedType) {
static AvroTypeException typeException(Deque<String> fieldPath, String expectedType, Object offendingValue) {
return new AvroTypeException(new StringBuilder()
.append("Field ")
.append(PathsPrinter.print(fieldPath))
.append(" is expected to be type: ")
.append(expectedType)
.append(", but it is: ")
.append(offendingValue)
.toString());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -289,15 +289,16 @@ private Object readUnion(Schema.Field field, Schema schema, Object value, Deque<
throw unionException(
field.name(),
types.stream().map(Schema::getType).map(Object::toString).collect(joining(", ")),
path);
path,
value);
}

private Object ensureEnum(Schema schema, Object value, Deque<String> path) {
List<String> symbols = schema.getEnumSymbols();
if (symbols.contains(value)) {
return new GenericData.EnumSymbol(schema, value);
}
throw enumException(path, symbols.stream().map(String::valueOf).collect(joining(", ")));
throw enumException(path, symbols.stream().map(String::valueOf).collect(joining(", ")), value);
}

private ByteBuffer bytesForString(String string) {
Expand All @@ -315,7 +316,7 @@ public <T> Object onValidType(Object value, Class<T> type, Deque<String> path, b
if (silently) {
return INCOMPATIBLE;
} else {
throw typeException(path, type.getTypeName());
throw typeException(path, type.getTypeName(), value);
}
}
}
Expand Down

0 comments on commit e007826

Please sign in to comment.