HDFS Java客户端Daemon

1.简单介绍

1.1.pom依赖

比较老的版本

<dependencyManagement><dependencies><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-common</artifactId><version>2.8.4</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-hdfs</artifactId><version>2.8.4</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-client</artifactId><version>2.8.4</version></dependency><dependency><groupId>org.projectlombok</groupId><artifactId>lombok</artifactId><version>1.16.10</version></dependency><dependency><groupId>log4j</groupId><artifactId>log4j</artifactId><version>1.2.17</version></dependency><dependency><groupId>org.slf4j</groupId><artifactId>slf4j-api</artifactId><version>1.7.7</version></dependency></dependencies>
</dependencyManagement>

1.2.简单介绍

org.apache.hadoop.fs.FileSystem是我们向HDFS发送请求的客户端，里面定义了HDFS常用的一些文件操作接口。

org.apache.hadoop.conf.Configuration，承载创建FileSystem客户端时的配置信息。

创建客户端的方法：

import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.conf.Configuration;import java.net.URI;// 最简单的配置，可以用来创建一些读取、下载文件的客户端
// 1.配置信息
Configuration config = new Configuration();
config.set("fs.defaultFS", "hdfs://node1:9000");  // HDFS服务端地址
//2.获取文件系统
FileSystem fs = FileSystem.get(config);//指定用户、文件副本数的配置，可以用来创建一些上传、创建的客户端
// 1.配置信息
Configuration config = new Configuration();
config.set("dfs.replication", "2");
//2.获取文件系统
FileSystem fs = FileSystem.get(new URI("hdfs://node1:9000"), config, "root");

常用接口：

1.上传


//1.上传本地Windows文件的路径
Path src = new Path("D:\\xxx");//2.要上传到HDFS的路径
Path dst = new Path("hdfs://node1:9000/xxx");//3.以拷贝的方式上传，从src -> dst
fs.copyFromLocalFile(src,dst);

2.下载

// 指定一下用户和副本数
FileSystem fs = FileSystem.get(new URI("hdfs://node1:9000"), config, "root"); //1.上传本地Windows文件的路径
Path local = new Path("D:\\xxx");//2.要上传到HDFS的路径
Path remote = new Path("hdfs://node1:9000/xxx");// 参数含义：是否删除远程文件  远程文件   本地文件  使用本地文件系统
fs.copyToLocalFile(false, remote, local, true);

2.搭建daemon

目录结构

2.1. ServerEnum.java

记录配置信息

package pool;import org.apache.hadoop.conf.Configuration;public enum ServerEnum {DEFAULT("hdfs://node1:9000") {@Overridepublic Configuration getConfig() {Configuration config = new Configuration();config.set("fs.defaultFS", url);return config;}},REPLICATE2("hdfs://node1:9000") {@Overridepublic Configuration getConfig() {Configuration config = new Configuration();config.set("dfs.replication", "2");return config;}};public final String url;ServerEnum(String url) {this.url = url;}public abstract Configuration getConfig();
}

2.1. HdfsClient.java

生产HDFS客户端

package pool;import org.apache.hadoop.fs.FileSystem;import java.net.URI;public class HdfsClient implements AutoCloseable{private final FileSystem fileSystem;public FileSystem getFileSystem() {return fileSystem;}private HdfsClient(FileSystem fileSystem) {this.fileSystem = fileSystem;}public static HdfsClient from(ServerEnum serverEnum) throws Exception {FileSystem fileSystem = FileSystem.get(serverEnum.getConfig());return new HdfsClient(fileSystem);}public static HdfsClient from(ServerEnum serverEnum, String user) throws Exception {FileSystem fileSystem = FileSystem.get(new URI(serverEnum.url), serverEnum.getConfig(), user);return new HdfsClient(fileSystem);}@Overridepublic void close() throws Exception {if (fileSystem != null) {fileSystem.close();}}
}

2.3. HdfsUtil.java

package hdfs01;import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import pool.HdfsClient;
import pool.ServerEnum;import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.nio.charset.StandardCharsets;public class HdfsUtil {private static void upload(String local, String remote) throws Exception {try (HdfsClient client = HdfsClient.from(ServerEnum.REPLICATE2, "root");FileSystem fs = client.getFileSystem()) {fs.copyFromLocalFile(new Path(local), new Path(remote));}System.out.println("upload success");}private static void download(String remote, String local) throws Exception {try (HdfsClient client = HdfsClient.from(ServerEnum.DEFAULT, "root");FileSystem fs = client.getFileSystem()) {fs.copyToLocalFile(false, new Path(remote), new Path(local), true);}System.out.println("download success");}private static void streamUpload(String local, String remote) throws Exception {try (HdfsClient client = HdfsClient.from(ServerEnum.REPLICATE2, "root");FileSystem fs = client.getFileSystem()) {FileInputStream input = new FileInputStream(new File(local));FSDataOutputStream output = fs.create(new Path(remote));IOUtils.copyBytes(input, output, 4*1024, false);}System.out.println("upload by stream success");}private static void streamDownload(String remote, String local) throws Exception {try (HdfsClient client = HdfsClient.from(ServerEnum.DEFAULT);FileSystem fs = client.getFileSystem()) {FileOutputStream out = new FileOutputStream(local);FSDataInputStream in = fs.open(new Path(remote));IOUtils.copyBytes(in, out, 4*1024, false);System.out.println("downlaod by stream success");}}private static void create(String file, byte[] data) throws Exception {try (HdfsClient client = HdfsClient.from(ServerEnum.REPLICATE2, "root");FileSystem fs = client.getFileSystem()) {FSDataOutputStream outPut = fs.create(new Path(file));outPut.write(data);outPut.close();}}public static void main(String[] args) throws Exception {String local1 = HdfsUtil.class.getResource("/hello.txt").getPath();String local2 = "D:\\code\\pratise\\big-data\\study\\hadoop\\src\\main\\resources\\down.txt";String remote1 = "/hello.txt";String remote2 = "hdfs://node1:9000/hello2.txt";String createFile = "hdfs://node1:9000/new.txt";//        upload(local, remote1;
//
//        download(remote2, "");//        streamUpload(local1, remote1);//            streamDownload(remote2, local2);create(createFile, "这是新建的文件".getBytes(StandardCharsets.UTF_8));}
}