一、首先自然是導(dǎo)包

? ?$HADOOP_HOME/share/hadoop/common/*.jar
? ?$HADOOP_HOME/share/hadoop/common/lib/*.jar
? ?$HADOOP_HOME/share/hadoop/hdfs/*.jar
? ?$HADOOP_HOME/share/hadoop/hdfs/lib/*.jar

---

二、代碼如下

package com.stu.hdfs; /*** * @author ysw28* HDFS的API操作*/import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.util.Arrays;import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.io.IOUtils; import org.junit.Test;public class TestDemo319 {@Test// 創(chuàng)建目錄public void mkDir() {try {// 連接HDFS,設(shè)置副本數(shù)Configuration conf = new Configuration();conf.set("dfs.replication", "1");// 創(chuàng)建一個客戶端FileSystem client = FileSystem.get(new URI("hdfs://HP110:9000"), conf, "root");// 創(chuàng)建目錄Path path = new Path("/Demo319");client.mkdirs(path);// 關(guān)閉客戶端client.close();System.out.println("創(chuàng)建目錄成功");} catch (Exception e) {e.printStackTrace();System.out.println("創(chuàng)建目錄失敗");}}@Test// 刪除目錄，刪除文件同理public void delDir() {try {// 連接HDFSConfiguration conf = new Configuration();// 創(chuàng)建客戶端FileSystem client = FileSystem.get(new URI("hdfs://HP110:9000"), conf, "root");// 刪除目錄Path path = new Path("/Demo319");if (client.exists(path)) {client.delete(path, true);// 關(guān)閉客戶端client.close();System.out.println("刪除目錄成功");} else {System.out.println("沒有這個文件");}} catch (Exception e) {e.printStackTrace();System.out.println("刪除目錄失敗");}}@Test// 上傳文件public void copyFromLocal() {try {// 連接HDFS，指定副本數(shù)Configuration conf = new Configuration();conf.set("dfs.replication", "1");// 創(chuàng)建客戶端FileSystem client = FileSystem.get(new URI("hdfs://Hp110:9000"), conf, "root");// 上傳文件client.copyFromLocalFile(new Path("C:\\Users\\ysw28\\Desktop\\hadoop-2.8.5.tar.gz"), new Path("/Demo319"));// 關(guān)閉客戶端client.close();System.out.println("上傳文件成功");} catch (Exception e) {e.printStackTrace();System.out.println("上傳文件失敗");}}@Test// 下載文件public void copyToFrom() {try {// 連接HDFSConfiguration conf = new Configuration();// 創(chuàng)建客戶端FileSystem client = FileSystem.get(new URI("hdfs://HP110:9000"), conf, "root");// 下載文件client.copyToLocalFile(true, new Path("/Demo319/hadoop-2.8.5.tar.gz"), new Path("E://"), true);// 關(guān)閉客戶端client.close();System.out.println("上傳文件成功");} catch (Exception e) {e.printStackTrace();System.out.println("上傳文件失敗");}}@Test// 使用HDFS的IO流上傳文件public void putFileToHdfs() {try {// 連接HDFS，指定副本數(shù)Configuration conf = new Configuration();conf.set("dfs.replication", "1");// 創(chuàng)建客戶端FileSystem client = FileSystem.get(new URI("hdfs://HP110:9000"), conf, "root");// 上傳文件，構(gòu)造一個輸入流InputStream input = new FileInputStream("E:\\hadoop-2.8.5.tar.gz");// 構(gòu)造一個輸出流OutputStream output = client.create(new Path("/Demo319/hadoop.tar.gz"));// 構(gòu)造一個緩存區(qū)byte buffer[] = new byte[1024];// 指定長度int len = 0;// 讀取數(shù)據(jù)while ((len = input.read(buffer)) > 0) {// 寫入數(shù)據(jù)output.write(buffer, 0, len);}// 刷新output.flush();// 關(guān)閉流output.close();input.close();// 關(guān)閉客戶端client.close();System.out.println("通過HDFS的IO流上傳文件成功");} catch (Exception e) {e.printStackTrace();System.out.println("通過HDFS的IO流上傳文件失敗");}}@Test// 使用HDFS的IO流下載文件public void getFileFromHdfs() {try {// 連接HDFSConfiguration conf = new Configuration();// 創(chuàng)建客戶端FileSystem client = FileSystem.get(new URI("hdfs://HP110:9000"), conf, "root");// 下載文件，構(gòu)造一個輸入流InputStream input = client.open(new Path("/Demo319/hadoop.tar.gz"));// 構(gòu)造一個輸出流OutputStream output = new FileOutputStream("E://1.tar.gz");// 使用工具類IOUtils.copyBytes(input, output, 1024);// 關(guān)閉客戶端client.close();System.out.println("通過HDFS的IO流下載文件成功");} catch (Exception e) {e.printStackTrace();System.out.println("通過HDFS的IO流下載文件失敗");}}@Test// 更改文件名，更改目錄名同理public void reName() {try {// 連接HDFSConfiguration conf = new Configuration();// 創(chuàng)建客戶端FileSystem client = FileSystem.get(new URI("hdfs://HP110:9000"), conf, "root");// 更改文件名Path oldpath = new Path("/Demo319/1.txt");Path newPath = new Path("/Demo319/2.txt");client.rename(oldpath, newPath);// 關(guān)閉客戶端client.close();System.out.println("更改文件名成功");} catch (Exception e) {e.printStackTrace();System.out.println("更改文件名失敗");}}@Test// 寫入數(shù)據(jù)public void writeFile() {try {// 連接HDFS，配置副本數(shù)Configuration conf = new Configuration();conf.set("dfs.replication", "1");// 創(chuàng)建客戶端FileSystem client = FileSystem.get(new URI("hdfs://HP110:9000"), conf, "root");// 寫入數(shù)據(jù)FSDataOutputStream output = client.create(new Path("/Demo319/3.txt"));output.writeUTF("我喜歡大數(shù)據(jù)");// 關(guān)閉流output.close();// 關(guān)閉客戶端client.close();System.out.println("寫入文件成功");} catch (Exception e) {e.printStackTrace();System.out.println("寫入文件失敗");}}@Test// 讀取文件public void readFile() {try {// 連接HDFSConfiguration conf = new Configuration();// 創(chuàng)建客戶端FileSystem client = FileSystem.get(new URI("hdfs://HP110:9000"), conf, "root");// 讀取文件Path path = new Path("/Demo319/3.txt");if (client.exists(path)) {InputStream input = client.open(path);// 構(gòu)造一個緩存區(qū)byte buffer[] = new byte[1024];int foot = 0;// 指定數(shù)組的下標(biāo)int temp = 0;// 指定接受每次讀取的字節(jié)數(shù)據(jù)while ((temp = input.read()) != -1) {buffer[foot++] = (byte) temp;}System.out.println(new String(buffer, 0, foot));// 關(guān)閉流input.close();// 關(guān)閉客戶端client.close();System.out.println("讀取文件成功");} else {System.out.println("沒有這個文件");}} catch (Exception e) {e.printStackTrace();System.out.println("讀取文件失敗");}}@Test// 判斷HDFS上是否存在某文件public void hdfsFileExists() {try {// 連接HDFS// 指定當(dāng)前使用的用戶是rootSystem.setProperty("HADOOP_USER_NAME", "root");// 配置參數(shù)，指定NameNode地址Configuration conf = new Configuration();conf.set("fs.defaultFS", "hdfs://HP110:9000");// 創(chuàng)建一個客戶端FileSystem client = FileSystem.get(conf);// 判斷是否存在某文件if (client.exists(new Path("/Demo/1.txt"))) {System.out.println("存在文件1.txt");} else {System.out.println("不存在文件1.txt");}System.out.println("沒有出現(xiàn)錯誤");} catch (Exception e) {e.printStackTrace();System.out.println("出現(xiàn)錯誤");}}@Test// 查看文件信息public void getFileInfo() {try {// 連接HDFSConfiguration conf = new Configuration();// 創(chuàng)建客戶端FileSystem client = FileSystem.get(new URI("hdfs://HP110:9000"), conf, "root");// 查看文件信息Path path = new Path("/");RemoteIterator<LocatedFileStatus> listFiles = client.listFiles(path, true);while (listFiles.hasNext()) {LocatedFileStatus status = listFiles.next();System.out.println(status.getPath().getName()); // 獲取文件名System.out.println(status.getGroup()); // 獲取所在組System.out.println(status.getLen());// 文件大小System.out.println(status.getPermission());// 獲取權(quán)限// 獲取數(shù)據(jù)塊信息BlockLocation[] blockLocations = status.getBlockLocations();for (BlockLocation blockLocation : blockLocations) {// 數(shù)據(jù)塊所在主機(jī)名String[] hosts = blockLocation.getHosts();for (String host : hosts) {System.out.println(host);}// 數(shù)據(jù)塊所在主機(jī)IPString[] names = blockLocation.getNames();for (String name : names) {System.out.println(name);}}System.out.println("這是一個分割線-----------------------------");}System.out.println("查看文件信息成功");} catch (Exception e) {e.printStackTrace();System.out.println("查看文件信息失敗");}}@Test// 獲取數(shù)據(jù)塊信息public void getBlockInfo() {try {// 連接HDFS// 指定使用的用戶是rootSystem.setProperty("HADOOP_USER_NAME", "root");// 配置參數(shù),指明NameNode的地址Configuration conf = new Configuration();conf.set("fs.defaultFS", "hdfs://HP110:9000");// 創(chuàng)建一個客戶端FileSystem client = FileSystem.get(conf);// 獲取文件status信息FileStatus fileStatus = client.getFileStatus(new Path("/Demo/100.txt"));// 獲取文件的數(shù)據(jù)塊信息BlockLocation locaktions[] = client.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());/** 將數(shù)組轉(zhuǎn)換為字符串 語法：Arrays.toString(數(shù)組名);*/for (BlockLocation blockLocation : locaktions) {System.out.println(Arrays.toString(blockLocation.getHosts()) + "\t" + Arrays.toString(blockLocation.getNames()));System.out.println(fileStatus);}// 關(guān)閉客戶端client.close();System.out.println("獲取數(shù)據(jù)塊信息成功");} catch (Exception e) {e.printStackTrace();System.out.println("獲取數(shù)據(jù)塊信息失敗");}}@Test// 查看HDFS中所有的文件和內(nèi)容，想看具體的話就在/后面累加文件的名字public void getListFileInfo() {try {// 連接HDFS// 指定當(dāng)前用戶是rootSystem.setProperty("HADOOP_USER_NAME", "root");// 配置參數(shù)，指定NameNode地址Configuration conf = new Configuration();conf.set("fs.defaultFS", "hdfs://HP110:9000");// 創(chuàng)建一個客戶端FileSystem client = FileSystem.get(conf);// 指定查看地址Path path = new Path("/Demo/100.txt");FileStatus list[] = client.listStatus(path);for (FileStatus fileStatus : list) {System.out.println(fileStatus);}System.out.println("查看文件成功");} catch (Exception e) {e.printStackTrace();System.out.println("查看文件失敗");}}}

總結(jié)

以上是生活随笔為你收集整理的HDFS使用JavaAPI操作上传特定副本到datanode的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯，歡迎將生活随笔推薦給好友。