Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

HADOOP-19334. Remove usage of sun.misc.Unsafe #7156

Open
wants to merge 1 commit into
base: trunk
Choose a base branch
from
+66 −216
Open

HADOOP-19334. Remove usage of sun.misc.Unsafe #7156

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 17 additions & 26 deletions ...-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@
package org.apache.hadoop.io;

import java.lang.reflect.Field;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.security.AccessController;
import java.security.PrivilegedAction;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sun.misc.Unsafe;

import org.apache.hadoop.thirdparty.com.google.common.primitives.UnsignedBytes;

Expand Down Expand Up @@ -58,7 +58,7 @@ private static Comparer<byte[]> lexicographicalComparerJavaImpl() {

/**
* Provides a lexicographical comparer implementation; either a Java
* implementation or a faster implementation based on {@link Unsafe}.
* implementation.
*
* <p>Uses reflection to gracefully fall back to the Java implementation if
* {@code Unsafe} isn't available.
Expand Down Expand Up @@ -131,34 +131,19 @@ public int compareTo(byte[] buffer1, int offset1, int length1,
private enum UnsafeComparer implements Comparer<byte[]> {
INSTANCE;

static final Unsafe theUnsafe;

/** The offset to the first element in a byte array. */
static final int BYTE_ARRAY_BASE_OFFSET;

static {
theUnsafe = (Unsafe) AccessController.doPrivileged(
new PrivilegedAction<Object>() {
@Override
public Object run() {
try {
Field f = Unsafe.class.getDeclaredField("theUnsafe");
f.setAccessible(true);
return f.get(null);
} catch (NoSuchFieldException e) {
// It doesn't matter what we throw;
// it's swallowed in getBestComparer().
throw new Error();
} catch (IllegalAccessException e) {
throw new Error();
}
}
});

BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class);
/**
* For a byte[] array, the base offset is always 0 because byte is the most fundamental data type in Java,
* and it does not have any additional header information.
* Therefore, it can be safely assumed that its base offset is 0.
*/
BYTE_ARRAY_BASE_OFFSET = 0;

// sanity check - this should never fail
if (theUnsafe.arrayIndexScale(byte[].class) != 1) {
if (Byte.BYTES != 1) {
throw new AssertionError();
}
}
Expand Down Expand Up @@ -207,8 +192,8 @@ public int compareTo(byte[] buffer1, int offset1, int length1,
* On the other hand, it is substantially faster on 64-bit.
*/
for (i = 0; i < strideLimit; i += stride) {
long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i);
long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i);
long lw = getLongFromByteBuffer(buffer1, offset1Adj + i);
long rw = getLongFromByteBuffer(buffer2, offset2Adj + i);

if (lw != rw) {
if (!littleEndian) {
Expand Down Expand Up @@ -240,6 +225,12 @@ public int compareTo(byte[] buffer1, int offset1, int length1,
}
return length1 - length2;
}

// Get the long value from the specified offset.
private static long getLongFromByteBuffer(byte[] buffer, int offset) {
ByteBuffer byteBuffer = ByteBuffer.wrap(buffer);
return byteBuffer.getLong(offset);
}
}
}
}
29 changes: 17 additions & 12 deletions ...op-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,7 @@
*/
package org.apache.hadoop.io.nativeio;

import java.io.File;
import java.io.FileDescriptor;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.lang.reflect.Field;
import java.io.*;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
Expand All @@ -45,7 +39,6 @@

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sun.misc.Unsafe;

import org.apache.hadoop.classification.VisibleForTesting;

Expand Down Expand Up @@ -897,11 +890,23 @@ static long getMemlockLimit() {
* @return the operating system's page size.
*/
static long getOperatingSystemPageSize() {
String os = System.getProperty("os.name").toLowerCase();
ProcessBuilder processBuilder;
if (os.contains("mac") || os.contains("linux") || os.contains("unix")) {
processBuilder = new ProcessBuilder("getconf", "PAGESIZE");
} else {
return 4096;
}
try {
Field f = Unsafe.class.getDeclaredField("theUnsafe");
f.setAccessible(true);
Unsafe unsafe = (Unsafe)f.get(null);
return unsafe.pageSize();
Process process = processBuilder.start();
try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
String line = reader.readLine();
if (line != null && !line.isEmpty()) {
return Long.parseLong(line.trim());
} else {
return 4096;
}
}
} catch (Throwable e) {
LOG.warn("Unable to get operating system page size. Guessing 4096.", e);
return 4096;
Expand Down
54 changes: 15 additions & 39 deletions ...hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@

import java.io.FileInputStream;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.BitSet;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.Random;
import java.util.concurrent.atomic.AtomicLong;

import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
Expand All @@ -37,8 +37,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import sun.misc.Unsafe;

import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.thirdparty.com.google.common.collect.ComparisonChain;
import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints;
Expand All @@ -54,19 +52,6 @@ public class ShortCircuitShm {

protected static final int BYTES_PER_SLOT = 64;

private static final Unsafe unsafe = safetyDance();

private static Unsafe safetyDance() {
try {
Field f = Unsafe.class.getDeclaredField("theUnsafe");
f.setAccessible(true);
return (Unsafe)f.get(null);
} catch (Throwable e) {
LOG.error("failed to load misc.Unsafe", e);
}
return null;
}

/**
* Calculate the usable size of a shared memory segment.
* We round down to a multiple of the slot size and do some validation.
Expand Down Expand Up @@ -261,15 +246,15 @@ public class Slot {
/**
* The slot address in memory.
*/
private final long slotAddress;
private final AtomicLong slotAddress; // Use AtomicLong instead of Unsafe

/**
* BlockId of the block this slot is used for.
*/
private final ExtendedBlockId blockId;

Slot(long slotAddress, ExtendedBlockId blockId) {
this.slotAddress = slotAddress;
this.slotAddress = new AtomicLong(slotAddress);
this.blockId = blockId;
}

Expand Down Expand Up @@ -307,41 +292,39 @@ public SlotId getSlotId() {
*/
public int getSlotIdx() {
return Ints.checkedCast(
(slotAddress - baseAddress) / BYTES_PER_SLOT);
(slotAddress.get() - baseAddress) / BYTES_PER_SLOT);
}

/**
* Clear the slot.
*/
void clear() {
unsafe.putLongVolatile(null, this.slotAddress, 0);
slotAddress.set(0);
}

private boolean isSet(long flag) {
long prev = unsafe.getLongVolatile(null, this.slotAddress);
long prev = slotAddress.get();
return (prev & flag) != 0;
}

private void setFlag(long flag) {
long prev;
do {
prev = unsafe.getLongVolatile(null, this.slotAddress);
prev = slotAddress.get();
if ((prev & flag) != 0) {
return;
}
} while (!unsafe.compareAndSwapLong(null, this.slotAddress,
prev, prev | flag));
} while (!slotAddress.compareAndSet(prev, prev | flag));
}

private void clearFlag(long flag) {
long prev;
do {
prev = unsafe.getLongVolatile(null, this.slotAddress);
prev = slotAddress.get();
if ((prev & flag) == 0) {
return;
}
} while (!unsafe.compareAndSwapLong(null, this.slotAddress,
prev, prev & (~flag)));
} while (!slotAddress.compareAndSet(prev, prev & (~flag)));
}

public boolean isValid() {
Expand Down Expand Up @@ -369,7 +352,7 @@ public void makeUnanchorable() {
}

public boolean isAnchored() {
long prev = unsafe.getLongVolatile(null, this.slotAddress);
long prev = slotAddress.get();
// Slot is no longer valid.
return (prev & VALID_FLAG) != 0 && ((prev & 0x7fffffff) != 0);
}
Expand All @@ -385,7 +368,7 @@ public boolean isAnchored() {
public boolean addAnchor() {
long prev;
do {
prev = unsafe.getLongVolatile(null, this.slotAddress);
prev = slotAddress.get();
if ((prev & VALID_FLAG) == 0) {
// Slot is no longer valid.
return false;
Expand All @@ -398,8 +381,7 @@ public boolean addAnchor() {
// Too many other threads have anchored the slot (2 billion?)
return false;
}
} while (!unsafe.compareAndSwapLong(null, this.slotAddress,
prev, prev + 1));
} while (!slotAddress.compareAndSet(prev, prev + 1));
return true;
}

Expand All @@ -409,12 +391,11 @@ public boolean addAnchor() {
public void removeAnchor() {
long prev;
do {
prev = unsafe.getLongVolatile(null, this.slotAddress);
prev = slotAddress.get();
Preconditions.checkState((prev & 0x7fffffff) != 0,
"Tried to remove anchor for slot " + slotAddress +", which was " +
"not anchored.");
} while (!unsafe.compareAndSwapLong(null, this.slotAddress,
prev, prev - 1));
} while (!slotAddress.compareAndSet(prev, prev - 1));
}

@Override
Expand Down Expand Up @@ -473,11 +454,6 @@ public ShortCircuitShm(ShmId shmId, FileInputStream stream)
throw new UnsupportedOperationException(
"DfsClientShm is not yet implemented for Windows.");
}
if (unsafe == null) {
throw new UnsupportedOperationException(
"can't use DfsClientShm because we failed to " +
"load misc.Unsafe.");
}
this.shmId = shmId;
this.mmappedLength = getUsableLength(stream);
this.baseAddress = POSIX.mmap(stream.getFD(),
Expand Down
Loading
Loading