## 环境搭建
* JAVA
* IDEA
* MAVEN
* pom.xml 要添加cdh仓库，然后maven install 非常慢。
~~~
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.bizzbee.bigdata</groupId>
<artifactId>hadoop-train-v2</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<hadoop.version>2.6.0-cdh5.15.1</hadoop.version>
</properties>
<!-- 添加仓库-->
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
~~~
## hdfs API测试
>单机的hadoop测试环境要加上下面这句配置。
> configuration.set("dfs.client.use.datanode.hostname", "true");
~~~
public class HDFSApp {
public static final String HDFS_PATH = "hdfs://tencent2:8020";
Configuration configuration;
FileSystem fileSystem;
@Before
public void setUp() throws URISyntaxException, IOException, InterruptedException {
configuration = new Configuration();
//我去，下面这句话相当重要啊！不然读取文件有问题啊！！！
configuration.set("dfs.client.use.datanode.hostname", "true");
fileSystem = FileSystem.get(new URI(HDFS_PATH),configuration,"bizzbee");
}
/*
* 读取文件
* */
@Test
public void text() throws IOException {
FSDataInputStream in =fileSystem.open(new Path("/bizzbee/test/cba"));
IOUtils.copyBytes(in,System.out,1024);
}
/*
* 创建文件
* */
@Test
public void create() throws Exception{
FSDataOutputStream out = fileSystem.create(new Path("/bizzbee/test/nba.txt"));
out.writeUTF("湖人总冠军！");
out.flush();
out.close();
}
/*
* 重命名
* */
@Test
public void rename() throws Exception{
Path oldP = new Path("/bizzbee/test/nba.txt");
Path newP = new Path("/bizzbee/test/cba.txt");
boolean r = fileSystem.rename(oldP,newP);
System.out.println(r);
}
/*
* 创建文件夹*/
@Test
public void mkdir () throws Exception{
fileSystem.mkdirs(new Path("/bizzbee/test"));
}
/*
* 拷贝本地文件到文件系统
* */
@Test
public void copyFromLocalFile() throws Exception{
Path from = new Path("/Users/bizzbee/IdeaProjects/hadooptrainv2/pom.xml");
Path to = new Path("/bizzbee/test/pom.xml");
fileSystem.copyFromLocalFile(from,to);
}
/*
* 拷贝本地大型文件
* 带进度条
* */
@Test
public void copyFromLocalBigFile() throws Exception{
//本地读一个大文件
InputStream in = new BufferedInputStream(new FileInputStream(new File("/Users/bizzbee/python/python-3.7.0-macosx10.9.pkg")));
//往hdfs输出流。
FSDataOutputStream out = fileSystem.create(new Path("/bizzbee/test/python-mac1.pkg"), new Progressable() {
public void progress() {
System.out.print(".");
}
});
IOUtils.copyBytes(in,out,4096);
}
/*
* 下载文件，拷贝到本地
* */
@Test
public void copyToLocalFile() throws Exception{
Path from = new Path("/bizzbee/test/cba.txt");
Path to = new Path("/Users/bizzbee/Desktop/cba.txt");
fileSystem.copyToLocalFile(from,to);
}
/*
* 列出所有文件
* */
@Test
public void listFiles() throws Exception{
FileStatus[] statuses = fileSystem.listStatus(new Path("/bizzbee/test"));
for (FileStatus file:statuses){
String isDir = file.isDirectory()? "file":"dir";
String permission = file.getPermission().toString();
short rep = file.getReplication();
long length = file.getLen();
String path = file.getPath().toString();
System.out.println(isDir+"--"+permission+"--"+rep+"--"+length+"--"+path);
}
}
/*
* 递归展示目录文件
* */
@Test
public void listFilesRecursive() throws Exception{
RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path("/bizzbee/test"),true);
while (files.hasNext()){
LocatedFileStatus file = files.next();
String isDir = file.isDirectory()? "file":"dir";
String permission = file.getPermission().toString();
short rep = file.getReplication();
long length = file.getLen();
String path = file.getPath().toString();
System.out.println(isDir+"--"+permission+"--"+rep+"--"+length+"--"+path);
}
}
/*块信息*/
@Test
public void getFileBlocKLocations()throws Exception{
FileStatus fileStatus = fileSystem.getFileStatus(new Path("/bizzbee/test/python-mac1.pkg"));
BlockLocation[] blocks = fileSystem.getFileBlockLocations(fileStatus,0,fileStatus.getLen());
for(BlockLocation block:blocks){
for(String name:block.getNames()) {
System.out.println(name + "----" + block.getOffset() + "----" + block.getLength() + "----" + block.getHosts().toString());
}
}
}
@Test
public void delete() throws Exception{
boolean result = fileSystem.delete(new Path("/bizzbee/test/python-mac.pkg"),true);
System.out.println(result);
}
@After
public void tearDown(){
configuration = null;
fileSystem = null;
System.out.println("tearDown");
}
}
~~~
