Skip to content

Commit

Permalink
fix: cleaning vector module v2
Browse files Browse the repository at this point in the history
  • Loading branch information
vibhatha committed Sep 11, 2024
1 parent 0e79ee9 commit de95e9c
Show file tree
Hide file tree
Showing 16 changed files with 71 additions and 36 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ public ZeroVector(Field field) {
}

@Deprecated
@SuppressWarnings("InlineMeInliner")
public ZeroVector() {}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -478,12 +478,15 @@ protected boolean compareBaseVariableWidthVectors(Range range) {
int offsetWidth = BaseVariableWidthVector.OFFSET_WIDTH;

if (!isNull) {
final int startIndexLeft = leftVector.getOffsetBuffer().getInt(leftIndex * offsetWidth);
final int endIndexLeft = leftVector.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth);
final int startIndexLeft =
leftVector.getOffsetBuffer().getInt((long) leftIndex * offsetWidth);
final int endIndexLeft =
leftVector.getOffsetBuffer().getInt((long) (leftIndex + 1) * offsetWidth);

final int startIndexRight = rightVector.getOffsetBuffer().getInt(rightIndex * offsetWidth);
final int startIndexRight =
rightVector.getOffsetBuffer().getInt((long) rightIndex * offsetWidth);
final int endIndexRight =
rightVector.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth);
rightVector.getOffsetBuffer().getInt((long) (rightIndex + 1) * offsetWidth);

int ret =
ByteFunctionHelpers.equal(
Expand Down Expand Up @@ -657,12 +660,15 @@ protected boolean compareListVectors(Range range) {
int offsetWidth = BaseRepeatedValueVector.OFFSET_WIDTH;

if (!isNull) {
final int startIndexLeft = leftVector.getOffsetBuffer().getInt(leftIndex * offsetWidth);
final int endIndexLeft = leftVector.getOffsetBuffer().getInt((leftIndex + 1) * offsetWidth);
final int startIndexLeft =
leftVector.getOffsetBuffer().getInt((long) leftIndex * offsetWidth);
final int endIndexLeft =
leftVector.getOffsetBuffer().getInt((long) (leftIndex + 1) * offsetWidth);

final int startIndexRight = rightVector.getOffsetBuffer().getInt(rightIndex * offsetWidth);
final int startIndexRight =
rightVector.getOffsetBuffer().getInt((long) rightIndex * offsetWidth);
final int endIndexRight =
rightVector.getOffsetBuffer().getInt((rightIndex + 1) * offsetWidth);
rightVector.getOffsetBuffer().getInt((long) (rightIndex + 1) * offsetWidth);

if ((endIndexLeft - startIndexLeft) != (endIndexRight - startIndexRight)) {
return false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,23 @@ public TypeEqualsVisitor(ValueVector right, boolean checkName, boolean checkMeta
}

/** Check type equals without passing IN param in VectorVisitor. */
public boolean equals(ValueVector left) {
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (!(obj instanceof ValueVector)) {
return false;
}
ValueVector left = (ValueVector) obj;
return left.accept(this, null);
}

@Override
public int hashCode() {
return Objects.hash(right, checkName, checkMetadata);
}

@Override
public Boolean visit(BaseFixedWidthVector left, Void value) {
return compareField(left.getField(), right.getField());
Expand Down Expand Up @@ -138,12 +151,12 @@ public Boolean visit(LargeListViewVector left, Void value) {

private boolean compareField(Field leftField, Field rightField) {

if (leftField == rightField) {
if (leftField.equals(rightField)) {
return true;
}

return (!checkName || Objects.equals(leftField.getName(), rightField.getName()))
&& Objects.equals(leftField.isNullable(), rightField.isNullable())
&& (leftField.isNullable() == rightField.isNullable())
&& Objects.equals(leftField.getType(), rightField.getType())
&& Objects.equals(leftField.getDictionary(), rightField.getDictionary())
&& (!checkMetadata || Objects.equals(leftField.getMetadata(), rightField.getMetadata()))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,10 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri

Mode mode = Mode.INIT;
private final String name;

@SuppressWarnings("UnusedVariable")
private final boolean unionEnabled;

private final NullableStructWriterFactory nullableStructWriterFactory;

private enum Mode {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ public StructOrListWriterImpl(final BaseWriter.ListWriter writer) {
}

/** Start writing to either the list or the struct. */
@Override
public void start() {
if (struct != null) {
struct.start();
Expand All @@ -60,6 +61,7 @@ public void start() {
}

/** Finish writing to the list or struct. */
@Override
public void end() {
if (struct != null) {
struct.end();
Expand All @@ -69,6 +71,7 @@ public void end() {
}

/** Creates a new writer for a struct with the given name. */
@Override
public StructOrListWriter struct(final String name) {
assert struct != null;
return new StructOrListWriterImpl(struct.struct(name));
Expand All @@ -81,6 +84,7 @@ public StructOrListWriter struct(final String name) {
* @deprecated use {@link #listOfStruct(String)} instead.
*/
@Deprecated
@SuppressWarnings({"InlineMeValidator", "InlineMeSuggester"})
public StructOrListWriter listoftstruct(final String name) {
return listOfStruct(name);
}
Expand All @@ -90,48 +94,59 @@ public StructOrListWriter listoftstruct(final String name) {
*
* @param name Unused.
*/
@Override
public StructOrListWriter listOfStruct(final String name) {
assert list != null;
return new StructOrListWriterImpl(list.struct());
}

@Override
public StructOrListWriter list(final String name) {
assert struct != null;
return new StructOrListWriterImpl(struct.list(name));
}

@Override
public boolean isStructWriter() {
return struct != null;
}

@Override
public boolean isListWriter() {
return list != null;
}

@Override
public VarCharWriter varChar(final String name) {
return (struct != null) ? struct.varChar(name) : list.varChar();
}

@Override
public IntWriter integer(final String name) {
return (struct != null) ? struct.integer(name) : list.integer();
}

@Override
public BigIntWriter bigInt(final String name) {
return (struct != null) ? struct.bigInt(name) : list.bigInt();
}

@Override
public Float4Writer float4(final String name) {
return (struct != null) ? struct.float4(name) : list.float4();
}

@Override
public Float8Writer float8(final String name) {
return (struct != null) ? struct.float8(name) : list.float8();
}

@Override
public BitWriter bit(final String name) {
return (struct != null) ? struct.bit(name) : list.bit();
}

@Override
public VarBinaryWriter binary(final String name) {
return (struct != null) ? struct.varBinary(name) : list.varBinary();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,7 @@ public int hashCode() {
return hash;
}

@Override
public final boolean equals(Object o) {
if (!(o instanceof DictionaryHashTable.Entry)) {
return false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ private static StructVector cloneVector(StructVector vector, BufferAllocator all
StructVector cloned =
(StructVector)
fieldType.createNewSingleVector(
vector.getField().getName(), allocator, /*schemaCallback=*/ null);
vector.getField().getName(), allocator, /* schemaCallBack= */ null);

final ArrowFieldNode fieldNode =
new ArrowFieldNode(vector.getValueCount(), vector.getNullCount());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ protected void closeReadSource() throws IOException {
* @return true if a batch was read, false on EOS
* @throws IOException on error
*/
@Override
public boolean loadNextBatch() throws IOException {
prepareLoadNextBatch();
MessageResult result = messageReader.readNext();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -155,15 +155,8 @@ protected void writeDictionaryBatch(Dictionary dictionary) throws IOException {
VectorUnloader unloader =
new VectorUnloader(dictRoot, /*includeNullCount*/ true, this.codec, /*alignBuffers*/ true);
ArrowRecordBatch batch = unloader.getRecordBatch();
ArrowDictionaryBatch dictionaryBatch = new ArrowDictionaryBatch(id, batch, false);
try {
try (ArrowDictionaryBatch dictionaryBatch = new ArrowDictionaryBatch(id, batch, false)) {
writeDictionaryBatch(dictionaryBatch);
} finally {
try {
dictionaryBatch.close();
} catch (Exception e) {
throw new RuntimeException("Error occurred while closing dictionary.", e);
}
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
Expand Down Expand Up @@ -248,14 +249,14 @@ private void writeFromVectorIntoJson(Field field, FieldVector vector) throws IOE
// writing views
ArrowBuf viewBuffer = vectorBuffers.get(1);
List<ArrowBuf> dataBuffers = vectorBuffers.subList(v + 1, vectorBuffers.size());
writeValueToViewGenerator(bufferType, viewBuffer, dataBuffers, vector, i);
writeValueToViewGenerator(viewBuffer, dataBuffers, vector, i);
} else if (bufferType.equals(VARIADIC_DATA_BUFFERS)
&& (vector.getMinorType() == MinorType.VIEWVARCHAR
|| vector.getMinorType() == MinorType.VIEWVARBINARY)) {
ArrowBuf viewBuffer = vectorBuffers.get(1); // check if this is v-1
List<ArrowBuf> dataBuffers = vectorBuffers.subList(v, vectorBuffers.size());
if (!dataBuffers.isEmpty()) {
writeValueToDataBufferGenerator(bufferType, viewBuffer, dataBuffers, vector);
writeValueToDataBufferGenerator(bufferType, viewBuffer, dataBuffers);
// The variadic buffers are written at once and doesn't require iterating for
// each index.
// So, break the loop.
Expand Down Expand Up @@ -350,7 +351,6 @@ private byte[] getView(final ArrowBuf viewBuffer, final List<ArrowBuf> dataBuffe
}

private void writeValueToViewGenerator(
BufferType bufferType,
ArrowBuf viewBuffer,
List<ArrowBuf> dataBuffers,
FieldVector vector,
Expand Down Expand Up @@ -383,7 +383,7 @@ private void writeValueToViewGenerator(
} else {
generator.writeFieldName("INLINED");
if (vector.getMinorType() == MinorType.VIEWVARCHAR) {
generator.writeString(new String(b, "UTF-8"));
generator.writeString(new String(b, StandardCharsets.UTF_8));
} else {
generator.writeString(Hex.encodeHexString(b));
}
Expand All @@ -392,7 +392,7 @@ private void writeValueToViewGenerator(
}

private void writeValueToDataBufferGenerator(
BufferType bufferType, ArrowBuf viewBuffer, List<ArrowBuf> dataBuffers, FieldVector vector)
BufferType bufferType, ArrowBuf viewBuffer, List<ArrowBuf> dataBuffers)
throws IOException {
if (bufferType.equals(VARIADIC_DATA_BUFFERS)) {
Preconditions.checkNotNull(viewBuffer);
Expand Down Expand Up @@ -560,8 +560,8 @@ private void writeValueToGenerator(
case VARCHAR:
{
Preconditions.checkNotNull(offsetBuffer);
byte[] b = (BaseVariableWidthVector.get(buffer, offsetBuffer, index));
generator.writeString(new String(b, "UTF-8"));
byte[] b = BaseVariableWidthVector.get(buffer, offsetBuffer, index);
generator.writeString(new String(b, StandardCharsets.UTF_8));
break;
}
case DECIMAL:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@
import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.vector.ipc.message.FBSerializable;
import org.apache.arrow.vector.ipc.message.MessageSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* Wrapper around a WritableByteChannel that maintains the position as well adding some common
Expand All @@ -37,7 +35,6 @@
* <p>Please note that objects of this class are not thread-safe.
*/
public class WriteChannel implements AutoCloseable {
private static final Logger LOGGER = LoggerFactory.getLogger(WriteChannel.class);

private static final byte[] ZERO_BYTES = new byte[8];

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
if (!(obj instanceof ArrowBlock)) {
return false;
}
ArrowBlock other = (ArrowBlock) obj;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
if (!(obj instanceof ArrowBuffer)) {
return false;
}
ArrowBuffer other = (ArrowBuffer) obj;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
*/
package org.apache.arrow.vector.ipc.message;

import com.google.errorprone.annotations.InlineMe;
import com.google.flatbuffers.FlatBufferBuilder;
import org.apache.arrow.flatbuf.DictionaryBatch;
import org.apache.arrow.flatbuf.MessageHeader;
Expand All @@ -31,6 +32,7 @@ public class ArrowDictionaryBatch implements ArrowMessage {
private final boolean isDelta;

@Deprecated
@InlineMe(replacement = "this(dictionaryId, dictionary, false)")
public ArrowDictionaryBatch(long dictionaryId, ArrowRecordBatch dictionary) {
this(dictionaryId, dictionary, false);
}
Expand All @@ -46,6 +48,7 @@ public boolean isDelta() {
return isDelta;
}

@Override
public byte getMessageType() {
return MessageHeader.DictionaryBatch;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
if (!(obj instanceof ArrowFooter)) {
return false;
}
ArrowFooter other = (ArrowFooter) obj;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@ public ArrowRecordBatch(
// this constructor is different from the public ones in that the reference manager's
// <code>retain</code> method is not called, so the first <code>dummy</code> parameter is used
// to distinguish this from the public constructor.
@SuppressWarnings("UnusedVariable")
private ArrowRecordBatch(
boolean dummy,
int length,
Expand All @@ -206,6 +207,7 @@ private ArrowRecordBatch(
this.buffersLayout = Collections.unmodifiableList(arrowBuffers);
}

@Override
public byte getMessageType() {
return org.apache.arrow.flatbuf.MessageHeader.RecordBatch;
}
Expand Down Expand Up @@ -261,9 +263,9 @@ public ArrowRecordBatch cloneWithTransfer(final BufferAllocator allocator) {
buffers.stream()
.map(
buf ->
(buf.getReferenceManager()
.transferOwnership(buf, allocator)
.getTransferredBuffer())
buf.getReferenceManager()
.transferOwnership(buf, allocator)
.getTransferredBuffer()
.writerIndex(buf.writerIndex()))
.collect(Collectors.toList());
close();
Expand Down

0 comments on commit de95e9c

Please sign in to comment.