使用HDFS JavaAP中的FileSystem类进行各种文件操作。
- 创建目录
- 查看目录
- 查看文件详情
- 复制本地文件到HDFS
- 剪切本地文件到HDFS
- 显示文件内容
- 追加一个文件到已经存在的文件的末尾
- 传输HDFS文件到本地
- 从HDFS一个路径剪切到HDFS的另一个路径
- 删除文件
- 删除目录
- 修改一个HDFS文件的副本数，并观察实际生效情况
程序能成功运行并完成指定任务。
/**
* 创建目录
*
* @throws IOException
*/
@Test
public void testMkdirs() throws IOException, URISyntaxException, InterruptedException {
Configuration configuration = new Configuration();
// 也可以通过设置运行时的VM参数 -DHADOOP_USER_NAME=root 来指定用户
FileSystem fs = FileSystem.get(new URI("hdfs://node71.xdata:8020"), configuration, "root");
fs.mkdirs(new Path("/user/root/output"));
fs.close();
}
/**
* 查看目录
*
* @throws IOException
* @throws InterruptedException
* @throws URISyntaxException
*/
@Test
public void testListStatus() throws IOException, InterruptedException, URISyntaxException {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://node71.xdata:8020"), configuration, "root");
FileStatus[] listStatus = fs.listStatus(new Path("/user/"));
for (FileStatus fileStatus : listStatus) {
if (fileStatus.isFile()) {
System.out.println("f "+fileStatus.getPath().getName());
}else {
System.out.println("d "+fileStatus.getPath().getName());
}
}
fs.close();
}
/**
* 查看文件详情
*
* @throws IOException
* @throws InterruptedException
* @throws URISyntaxException
*/
@Test
public void testListFiles() throws IOException, InterruptedException, URISyntaxException {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://node71.xdata:8020"), configuration, "root");
RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/user/root"), true);
while (listFiles.hasNext()) {
LocatedFileStatus status = listFiles.next();
// 权限
System.out.print(status.getPermission());
System.out.print("\t");
// 分组
System.out.print(status.getGroup());
System.out.print("\t");
// 长度
System.out.print(status.getLen());
System.out.print("\t");
// 文件名称
System.out.print(status.getPath().getName());
System.out.print("\t");
// 获取存储的块信息
BlockLocation[] blockLocations = status.getBlockLocations();
for (BlockLocation blockLocation : blockLocations) {
// 获取块存储的主机节点
String[] hosts = blockLocation.getHosts();
for (String host : hosts) {
System.out.print(host);
System.out.print("\t");
}
}
System.out.println();
}
// 3 关闭资源
fs.close();
}
/**
* 复制本地文件到HDFS
*
* @throws IOException
* @throws InterruptedException
* @throws URISyntaxException
*/
@Test
public void testCopyFromLocalFile() throws IOException, InterruptedException, URISyntaxException {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://node71.xdata:8020"), configuration, "root");
fs.copyFromLocalFile(new Path("D:\\data.txt"), new Path("/user/root/input/"));
fs.close();
}
/**
* 剪切本地文件到HDFS
*
* @throws IOException
* @throws InterruptedException
* @throws URISyntaxException
*/
@Test
public void testMoveFromLocalFile() throws IOException, InterruptedException, URISyntaxException {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://node71.xdata:8020"), configuration, "root");
fs.moveFromLocalFile(new Path("D:\\1562308284.png"), new Path("/user/root/input/1562308284_2.png"));
fs.close();
}
/**
* 显示文件内容
*
* @throws IOException
* @throws InterruptedException
* @throws URISyntaxException
*/
@Test
public void testOpen() throws IOException, InterruptedException, URISyntaxException {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://node71.xdata:8020"), configuration, "root");
InputStream in = null;
try {
in = fs.open(new Path(new URI("/user/root/input/data.txt")));
IOUtils.copyBytes(in, System.out, 4096, false);
} finally {
IOUtils.closeStream(in);
}
fs.close();
}
/**
* 追加一个文件到已经存在的文件的末尾
*
* @throws IOException
* @throws InterruptedException
* @throws URISyntaxException
*/
@Test
public void testAppend() throws IOException, InterruptedException, URISyntaxException {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://node71.xdata:8020"), configuration, "root");
InputStream in = new BufferedInputStream(new FileInputStream("D:\\data1.txt"));
OutputStream out = fs.append(new Path("/user/root/input/data.txt"));
IOUtils.copyBytes(in, out, 4096, true);
fs.close();
}
/**
* 传输HDFS文件到本地
*
* @throws IOException
* @throws InterruptedException
* @throws URISyntaxException
*/
@Test
public void testCopyToLocalFile() throws IOException, InterruptedException, URISyntaxException {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://node71.xdata:8020"), configuration, "root");
fs.copyToLocalFile(new Path("/user/root/1562308284.png"), new Path("D:\\1562308284.png"));
fs.close();
}
/**
* 从HDFS一个路径剪切到HDFS的另一个路径
*
* @throws IOException
* @throws InterruptedException
* @throws URISyntaxException
*/
@Test
public void testRename() throws IOException, InterruptedException, URISyntaxException {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://node71.xdata:8020"), configuration, "root");
fs.rename(new Path("/user/root/1562308284.png"), new Path("/user/root/output/2.png"));
fs.close();
}
/**
* 删除文件
*
* @throws IOException
* @throws InterruptedException
* @throws URISyntaxException
*/
@Test
public void testDeleteFile() throws IOException, InterruptedException, URISyntaxException {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://node71.xdata:8020"), configuration, "root");
fs.delete(new Path("/user/root/output/1562308284.png"), true);
fs.close();
}
/**
* 删除目录
*
* @throws IOException
* @throws InterruptedException
* @throws URISyntaxException
*/
@Test
public void testDeleteDir() throws IOException, InterruptedException, URISyntaxException {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://node71.xdata:8020"), configuration, "root");
// 删除非空目录时需要指定recursive为true 否则会抛出异常
fs.delete(new Path("/user/root/output"), true);
fs.close();
}
/**
* 修改一个HDFS文件的副本数，并观察实际生效情况
*
* @throws IOException
* @throws InterruptedException
* @throws URISyntaxException
*/
@Test
public void testSetReplication() throws IOException, InterruptedException, URISyntaxException {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(new URI("hdfs://node71.xdata:8020"), configuration, "root");
fs.setReplication(new Path("/user/root/input/1562308284.png"), (short)2);
fs.close();
}